From a9bc7dc5ca884834abbfeb2c30c9c7020ab0f091 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 13 Apr 2015 13:45:32 -0400 Subject: [PATCH 1/4] generate separate index for tools in repositories --- scripts/tool_shed/build_ts_whoosh_index.py | 203 +++++++++++++-------- 1 file changed, 122 insertions(+), 81 deletions(-) diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index a782a63b977..f3ad4d616b8 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -1,56 +1,63 @@ -# !/usr/bin/env python -""" Build indeces for searching the TS """ +""" Build indexes for searching the TS """ import sys import os -import csv -import urllib -import urllib2 import ConfigParser - + new_path = [ os.path.join( os.getcwd(), "lib" ) ] -new_path.extend( sys.path[1:] ) # remove scripts/ from the path +new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy.util import pretty_print_time_interval +import galaxy.webapps.tool_shed.model.mapping +from galaxy.webapps.tool_shed import config, model + from galaxy import eggs eggs.require( "SQLAlchemy" ) +eggs.require( "Whoosh" ) +from whoosh.filedb.filestore import FileStorage +from whoosh.fields import Schema, STORED, TEXT -import galaxy.webapps.tool_shed.model.mapping +repo_schema = Schema( + id=STORED, + name=TEXT( stored=True ), + description=TEXT( stored=True ), + long_description=TEXT( stored=True ), + homepage_url=TEXT( stored=True ), + remote_repository_url=TEXT( stored=True ), + repo_owner_username=TEXT( stored=True ), + times_downloaded=STORED, + approved=STORED, + last_updated=STORED, + full_last_updated=STORED ) -# Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether search is enabled. -try: - eggs.require( "Whoosh" ) - import whoosh.index - from whoosh.filedb.filestore import FileStorage - from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED - from whoosh.scoring import BM25F - from whoosh.qparser import MultifieldParser - from whoosh.index import Index - from galaxy.webapps.tool_shed import config, model - - whoosh_ready = True - schema = Schema( - id = STORED, - name = TEXT( field_boost = 1.7, stored = True ), - description = TEXT( field_boost = 1.5, stored = True ), - long_description = TEXT( stored = True ), - homepage_url = TEXT( stored = True ), - remote_repository_url = TEXT( stored = True ), - repo_owner_username = TEXT( stored = True ), - times_downloaded = STORED, - approved = STORED, - last_updated = STORED, - full_last_updated = STORED ) +tool_schema = Schema( + name=TEXT( stored=True ), + description=TEXT( stored=True ), + owner=TEXT( stored=True ), + id=TEXT( stored=True ), + help=TEXT( stored=True ), + version=TEXT( stored=True), + repo_owner_username=TEXT( stored=True ), + repo_id=STORED ) -except ImportError, e: - print 'import error' - whoosh_ready = False - schema = None def build_index( sa_session, toolshed_whoosh_index_dir ): - storage = FileStorage( toolshed_whoosh_index_dir ) - index = storage.create_index( schema ) - writer = index.writer() + """ + Build the search indexes. One for repositories and another for tools within. + """ + repo_index_storage = FileStorage( toolshed_whoosh_index_dir ) + tool_index_dir = os.path.join( toolshed_whoosh_index_dir, 'tools' ) + # Rare race condition exists here + if not os.path.exists( tool_index_dir ): + os.makedirs( tool_index_dir ) + tool_index_storage = FileStorage( tool_index_dir ) + + repo_index = repo_index_storage.create_index( repo_schema ) + tool_index = tool_index_storage.create_index( tool_schema ) + + repo_index_writer = repo_index.writer() + tool_index_writer = tool_index.writer() + def to_unicode( a_basestr ): if type( a_basestr ) is str: return unicode( a_basestr, 'utf-8' ) @@ -58,42 +65,56 @@ def build_index( sa_session, toolshed_whoosh_index_dir ): return a_basestr repos_indexed = 0 - for ( id, - name, - description, - long_description, - homepage_url, - remote_repository_url, - repo_owner_username, - times_downloaded, - approved, - last_updated, - full_last_updated ) in get_repos( sa_session ): + tools_indexed = 0 + + for repo in get_repos( sa_session ): + + repo_index_writer.add_document( id=repo.get( 'id' ), + name=to_unicode( repo.get( 'name' ) ), + description=to_unicode( repo.get( 'description' ) ), + long_description=to_unicode( repo.get( 'long_description' ) ), + homepage_url=to_unicode( repo.get( 'homepage_url' ) ), + remote_repository_url=to_unicode( repo.get( 'remote_repository_url' ) ), + repo_owner_username=to_unicode( repo.get( 'repo_owner_username' ) ), + times_downloaded=repo.get( 'times_downloaded' ), + approved=repo.get( 'approved' ), + last_updated=repo.get( 'last_updated' ), + full_last_updated=repo.get( 'full_last_updated' ) ) + # Tools get their own index + for tool in repo.get( 'tools_dict' ): + # print tool + tool_index_writer.add_document( id=to_unicode( tool.get( 'id' ) ), + name=to_unicode( tool.get( 'name' ) ), + version=to_unicode( tool.get( 'version' ) ), + description=to_unicode( tool.get( 'description' ) ), + help=to_unicode( tool.get( 'help' ) ), + repo_owner_username=to_unicode( repo.get( 'repo_owner_username' ) ), + repo_id=repo.get( 'id' ) ) + tools_indexed += 1 - writer.add_document( id = id, - name = to_unicode( name ), - description = to_unicode( description ), - long_description = to_unicode( long_description ), - homepage_url = to_unicode( homepage_url ), - remote_repository_url = to_unicode( remote_repository_url ), - repo_owner_username = to_unicode( repo_owner_username ), - times_downloaded = times_downloaded, - approved = approved, - last_updated = last_updated, - full_last_updated = full_last_updated ) repos_indexed += 1 - writer.commit() + + tool_index_writer.commit() + repo_index_writer.commit() + print "Number of repos indexed: ", repos_indexed + print "Number of tools indexed: ", tools_indexed + def get_repos( sa_session ): + """ + Load repos from DB + """ + results = [] for repo in sa_session.query( model.Repository ).filter_by( deleted=False ).filter_by( deprecated=False ).filter( model.Repository.type != 'tool_dependency_definition' ): - id = repo.id + + repo_id = repo.id name = repo.name description = repo.description long_description = repo.long_description homepage_url = repo.homepage_url remote_repository_url = repo.remote_repository_url - + times_downloaded = repo.times_downloaded if not isinstance( times_downloaded, ( int, long ) ): times_downloaded = 0 @@ -109,21 +130,42 @@ def get_repos( sa_session ): approved = 'yes' break - # Format the time since last update to be nicely readable. + # Format the time since last update to be nicely readable. last_updated = pretty_print_time_interval( repo.update_time ) full_last_updated = repo.update_time.strftime( "%Y-%m-%d %I:%M %p" ) - yield ( id, - name, - description, - long_description, - homepage_url, - remote_repository_url, - repo_owner_username, - times_downloaded, - approved, - last_updated, - full_last_updated ) + # Parse all the tools within repo for separate index. + tools_dict = [] + from galaxy.webapps.tool_shed.model import directory_hash_id + path = os.path.join( '/Users/marten/devel/git/galaxy/database/community_files', *directory_hash_id( repo.id )) + path = os.path.join( path, "repo_%d" % repo.id ) + from galaxy.tools.loader_directory import load_tool_elements_from_path + if os.path.exists(path): + tool_elems = load_tool_elements_from_path(path) + if tool_elems: + for elem in tool_elems: + root = elem[1].getroot() + if root.tag == 'tool': + tools_dict.append( dict( id=root.attrib.get('id'), + name=root.attrib.get('name'), + version=root.attrib.get('version'), + description=root.find('description').text, + help=root.find('help').text ) ) + + results.append(dict( id=repo_id, + name=name, + description=description, + long_description=long_description, + homepage_url=homepage_url, + remote_repository_url=remote_repository_url, + repo_owner_username=repo_owner_username, + times_downloaded=times_downloaded, + approved=approved, + last_updated=last_updated, + full_last_updated=full_last_updated, + tools_dict=tools_dict ) ) + return results + def get_sa_session_and_needed_config_settings( ini_file ): conf_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } ) @@ -139,8 +181,7 @@ def get_sa_session_and_needed_config_settings( ini_file ): return model.context.current, config_settings if __name__ == "__main__": - if whoosh_ready: - ini_file = sys.argv[ 1 ] - sa_session, config_settings = get_sa_session_and_needed_config_settings( ini_file ) - toolshed_whoosh_index_dir = config_settings.get( 'toolshed_whoosh_index_dir', None ) - build_index( sa_session, toolshed_whoosh_index_dir ) + ini_file = sys.argv[ 1 ] + sa_session, config_settings = get_sa_session_and_needed_config_settings( ini_file ) + toolshed_whoosh_index_dir = config_settings.get( 'toolshed_whoosh_index_dir', None ) + build_index( sa_session, toolshed_whoosh_index_dir ) From c12b8063e355f4dc365f4209f1385d41837de7f2 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 13 Apr 2015 14:39:31 -0400 Subject: [PATCH 2/4] remove abspath --- scripts/tool_shed/build_ts_whoosh_index.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index f3ad4d616b8..f2f72a13628 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -137,7 +137,8 @@ def get_repos( sa_session ): # Parse all the tools within repo for separate index. tools_dict = [] from galaxy.webapps.tool_shed.model import directory_hash_id - path = os.path.join( '/Users/marten/devel/git/galaxy/database/community_files', *directory_hash_id( repo.id )) + path = os.path.join( os.getcwd(), 'database/community_files' ) + path = os.path.join( path, *directory_hash_id( repo.id )) path = os.path.join( path, "repo_%d" % repo.id ) from galaxy.tools.loader_directory import load_tool_elements_from_path if os.path.exists(path): From 66e61271b1389959e6d1175c43091136495270de Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 14 Apr 2015 10:55:34 -0400 Subject: [PATCH 3/4] add comments and example; add consistency checks; remove bash script --- scripts/tool_shed/build_ts_whoosh_index.py | 43 +++++++++++++++------- scripts/tool_shed/build_ts_whoosh_index.sh | 6 --- 2 files changed, 30 insertions(+), 19 deletions(-) delete mode 100644 scripts/tool_shed/build_ts_whoosh_index.sh diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index f2f72a13628..178496a3bd4 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -1,4 +1,13 @@ -""" Build indexes for searching the TS """ +""" +Build indexes for searching the TS. +Run this script from the galaxy folder, example: + +$ python scripts/tool_shed/build_ts_whoosh_index_test.py config/tool_shed.ini + +Make sure you adjusted your config to: + * turn on searching via toolshed_search_on + * specify toolshed_whoosh_index_dir where the indexes will be placed +""" import sys import os import ConfigParser @@ -81,7 +90,7 @@ def build_index( sa_session, toolshed_whoosh_index_dir ): last_updated=repo.get( 'last_updated' ), full_last_updated=repo.get( 'full_last_updated' ) ) # Tools get their own index - for tool in repo.get( 'tools_dict' ): + for tool in repo.get( 'tools_list' ): # print tool tool_index_writer.add_document( id=to_unicode( tool.get( 'id' ) ), name=to_unicode( tool.get( 'name' ) ), @@ -91,19 +100,21 @@ def build_index( sa_session, toolshed_whoosh_index_dir ): repo_owner_username=to_unicode( repo.get( 'repo_owner_username' ) ), repo_id=repo.get( 'id' ) ) tools_indexed += 1 + print tools_indexed, 'tools (', tool.get( 'id' ), ')' repos_indexed += 1 + print repos_indexed, 'repos (', repo.get( 'id' ), ')' tool_index_writer.commit() repo_index_writer.commit() - print "Number of repos indexed: ", repos_indexed - print "Number of tools indexed: ", tools_indexed + print "TOTAL repos indexed: ", repos_indexed + print "TOTAL tools indexed: ", tools_indexed def get_repos( sa_session ): """ - Load repos from DB + Load repos from DB and included tools from .xml configs. """ results = [] for repo in sa_session.query( model.Repository ).filter_by( deleted=False ).filter_by( deprecated=False ).filter( model.Repository.type != 'tool_dependency_definition' ): @@ -135,9 +146,10 @@ def get_repos( sa_session ): full_last_updated = repo.update_time.strftime( "%Y-%m-%d %I:%M %p" ) # Parse all the tools within repo for separate index. - tools_dict = [] + tools_list = [] from galaxy.webapps.tool_shed.model import directory_hash_id - path = os.path.join( os.getcwd(), 'database/community_files' ) + path = '/srv/toolshed/test/var/data/repos' + # path = os.path.join( os.getcwd(), 'database/community_files' ) path = os.path.join( path, *directory_hash_id( repo.id )) path = os.path.join( path, "repo_%d" % repo.id ) from galaxy.tools.loader_directory import load_tool_elements_from_path @@ -147,11 +159,16 @@ def get_repos( sa_session ): for elem in tool_elems: root = elem[1].getroot() if root.tag == 'tool': - tools_dict.append( dict( id=root.attrib.get('id'), - name=root.attrib.get('name'), - version=root.attrib.get('version'), - description=root.find('description').text, - help=root.find('help').text ) ) + tool = {} + if root.find('help') is not None: + tool.update( dict( help=root.find( 'help' ).text ) ) + if root.find('description') is not None: + tool.update( dict( description=root.find( 'description' ).text ) ) + tool.update( dict( id=root.attrib.get('id'), + name=root.attrib.get('name'), + version=root.attrib.get('version') ) ) + + tools_list.append( tool ) results.append(dict( id=repo_id, name=name, @@ -164,7 +181,7 @@ def get_repos( sa_session ): approved=approved, last_updated=last_updated, full_last_updated=full_last_updated, - tools_dict=tools_dict ) ) + tools_list=tools_list ) ) return results diff --git a/scripts/tool_shed/build_ts_whoosh_index.sh b/scripts/tool_shed/build_ts_whoosh_index.sh deleted file mode 100644 index 37fbeab09cf..00000000000 --- a/scripts/tool_shed/build_ts_whoosh_index.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -cd `dirname $0`/../.. -# Make sure your config is at config/tool_shed.ini -# and that you specified toolshed_whoosh_index_dir in it. -python scripts/tool_shed/build_ts_whoosh_index.py config/tool_shed.ini From dc736c73f2f00df065200de20384b7de21059866 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 14 Apr 2015 11:31:44 -0400 Subject: [PATCH 4/4] remove another abspath; use optparse; --- scripts/tool_shed/build_ts_whoosh_index.py | 36 ++++++++++++---------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index 178496a3bd4..40f0ced29ab 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -1,8 +1,8 @@ """ Build indexes for searching the TS. -Run this script from the galaxy folder, example: +Run this script from the Tool Shed folder, example: -$ python scripts/tool_shed/build_ts_whoosh_index_test.py config/tool_shed.ini +$ python scripts/tool_shed/build_ts_whoosh_index_test.py -c config/tool_shed.ini -r database/community_files Make sure you adjusted your config to: * turn on searching via toolshed_search_on @@ -11,6 +11,7 @@ Make sure you adjusted your config to: import sys import os import ConfigParser +from optparse import OptionParser new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path @@ -18,7 +19,9 @@ sys.path = new_path from galaxy.util import pretty_print_time_interval import galaxy.webapps.tool_shed.model.mapping -from galaxy.webapps.tool_shed import config, model +from galaxy.webapps.tool_shed import config +from galaxy.webapps.tool_shed import model +from galaxy.tools.loader_directory import load_tool_elements_from_path from galaxy import eggs eggs.require( "SQLAlchemy" ) @@ -50,7 +53,7 @@ tool_schema = Schema( repo_id=STORED ) -def build_index( sa_session, toolshed_whoosh_index_dir ): +def build_index( sa_session, toolshed_whoosh_index_dir, path_to_repositories ): """ Build the search indexes. One for repositories and another for tools within. """ @@ -76,7 +79,7 @@ def build_index( sa_session, toolshed_whoosh_index_dir ): repos_indexed = 0 tools_indexed = 0 - for repo in get_repos( sa_session ): + for repo in get_repos( sa_session, path_to_repositories ): repo_index_writer.add_document( id=repo.get( 'id' ), name=to_unicode( repo.get( 'name' ) ), @@ -112,7 +115,7 @@ def build_index( sa_session, toolshed_whoosh_index_dir ): print "TOTAL tools indexed: ", tools_indexed -def get_repos( sa_session ): +def get_repos( sa_session, path_to_repositories ): """ Load repos from DB and included tools from .xml configs. """ @@ -147,12 +150,8 @@ def get_repos( sa_session ): # Parse all the tools within repo for separate index. tools_list = [] - from galaxy.webapps.tool_shed.model import directory_hash_id - path = '/srv/toolshed/test/var/data/repos' - # path = os.path.join( os.getcwd(), 'database/community_files' ) - path = os.path.join( path, *directory_hash_id( repo.id )) + path = os.path.join( path_to_repositories, *model.directory_hash_id( repo.id )) path = os.path.join( path, "repo_%d" % repo.id ) - from galaxy.tools.loader_directory import load_tool_elements_from_path if os.path.exists(path): tool_elems = load_tool_elements_from_path(path) if tool_elems: @@ -185,9 +184,9 @@ def get_repos( sa_session ): return results -def get_sa_session_and_needed_config_settings( ini_file ): +def get_sa_session_and_needed_config_settings( path_to_tool_shed_config ): conf_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } ) - conf_parser.read( ini_file ) + conf_parser.read( path_to_tool_shed_config ) kwds = dict() for key, value in conf_parser.items( "app:main" ): kwds[ key ] = value @@ -199,7 +198,12 @@ def get_sa_session_and_needed_config_settings( ini_file ): return model.context.current, config_settings if __name__ == "__main__": - ini_file = sys.argv[ 1 ] - sa_session, config_settings = get_sa_session_and_needed_config_settings( ini_file ) + parser = OptionParser() + parser.add_option("-c", "--config", dest="path_to_tool_shed_config", default="config/tool_shed.ini", help="specify tool_shed.ini location") + parser.add_option("-r", "--repos", dest="path_to_repositories", default="database/community_files", help="specify path to folder with repositories") + (options, args) = parser.parse_args() + path_to_tool_shed_config = options.path_to_tool_shed_config + path_to_repositories = options.path_to_repositories + sa_session, config_settings = get_sa_session_and_needed_config_settings( path_to_tool_shed_config ) toolshed_whoosh_index_dir = config_settings.get( 'toolshed_whoosh_index_dir', None ) - build_index( sa_session, toolshed_whoosh_index_dir ) + build_index( sa_session, toolshed_whoosh_index_dir, path_to_repositories )