mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-28 02:58:18 +08:00
222 lines
9.3 KiB
Python
222 lines
9.3 KiB
Python
"""
|
|
Build indexes for searching the TS.
|
|
Run this script from the Tool Shed folder, example:
|
|
|
|
$ python scripts/tool_shed/build_ts_whoosh_index.py -c config/tool_shed.ini
|
|
|
|
Make sure you adjusted your config to:
|
|
* turn on searching via toolshed_search_on
|
|
* specify whoosh_index_dir where the indexes will be placed
|
|
|
|
Also make sure that GALAXY_EGGS_PATH variable is properly set
|
|
in case you are using non-default location for Galaxy.
|
|
"""
|
|
import ConfigParser
|
|
import os
|
|
import sys
|
|
from optparse import OptionParser
|
|
|
|
from whoosh.fields import Schema, STORED, TEXT
|
|
from whoosh.filedb.filestore import FileStorage
|
|
|
|
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
|
|
|
|
import galaxy.webapps.tool_shed.model.mapping
|
|
from galaxy.tools.loader_directory import load_tool_elements_from_path
|
|
from galaxy.util import pretty_print_time_interval
|
|
from galaxy.webapps.tool_shed import config, model
|
|
|
|
repo_schema = Schema(
|
|
id=STORED,
|
|
name=TEXT( stored=True ),
|
|
description=TEXT( stored=True ),
|
|
long_description=TEXT( stored=True ),
|
|
homepage_url=TEXT( stored=True ),
|
|
remote_repository_url=TEXT( stored=True ),
|
|
repo_owner_username=TEXT( stored=True ),
|
|
times_downloaded=STORED,
|
|
approved=STORED,
|
|
last_updated=STORED,
|
|
full_last_updated=STORED )
|
|
|
|
tool_schema = Schema(
|
|
name=TEXT( stored=True ),
|
|
description=TEXT( stored=True ),
|
|
owner=TEXT( stored=True ),
|
|
id=TEXT( stored=True ),
|
|
help=TEXT( stored=True ),
|
|
version=TEXT( stored=True),
|
|
repo_name=TEXT( stored=True ),
|
|
repo_owner_username=TEXT( stored=True ),
|
|
repo_id=STORED )
|
|
|
|
|
|
def build_index( sa_session, whoosh_index_dir, path_to_repositories ):
|
|
"""
|
|
Build the search indexes. One for repositories and another for tools within.
|
|
"""
|
|
# Rare race condition exists here and below
|
|
if not os.path.exists( whoosh_index_dir ):
|
|
os.makedirs( whoosh_index_dir )
|
|
tool_index_dir = os.path.join( whoosh_index_dir, 'tools' )
|
|
if not os.path.exists( tool_index_dir ):
|
|
os.makedirs( tool_index_dir )
|
|
|
|
repo_index_storage = FileStorage( whoosh_index_dir )
|
|
tool_index_storage = FileStorage( tool_index_dir )
|
|
|
|
repo_index = repo_index_storage.create_index( repo_schema )
|
|
tool_index = tool_index_storage.create_index( tool_schema )
|
|
|
|
repo_index_writer = repo_index.writer()
|
|
tool_index_writer = tool_index.writer()
|
|
|
|
def to_unicode( a_basestr ):
|
|
if type( a_basestr ) is str:
|
|
return unicode( a_basestr, 'utf-8' )
|
|
else:
|
|
return a_basestr
|
|
|
|
repos_indexed = 0
|
|
tools_indexed = 0
|
|
|
|
for repo in get_repos( sa_session, path_to_repositories ):
|
|
|
|
repo_index_writer.add_document( id=repo.get( 'id' ),
|
|
name=to_unicode( repo.get( 'name' ) ),
|
|
description=to_unicode( repo.get( 'description' ) ),
|
|
long_description=to_unicode( repo.get( 'long_description' ) ),
|
|
homepage_url=to_unicode( repo.get( 'homepage_url' ) ),
|
|
remote_repository_url=to_unicode( repo.get( 'remote_repository_url' ) ),
|
|
repo_owner_username=to_unicode( repo.get( 'repo_owner_username' ) ),
|
|
times_downloaded=repo.get( 'times_downloaded' ),
|
|
approved=repo.get( 'approved' ),
|
|
last_updated=repo.get( 'last_updated' ),
|
|
full_last_updated=repo.get( 'full_last_updated' ) )
|
|
# Tools get their own index
|
|
for tool in repo.get( 'tools_list' ):
|
|
# print tool
|
|
tool_index_writer.add_document( id=to_unicode( tool.get( 'id' ) ),
|
|
name=to_unicode( tool.get( 'name' ) ),
|
|
version=to_unicode( tool.get( 'version' ) ),
|
|
description=to_unicode( tool.get( 'description' ) ),
|
|
help=to_unicode( tool.get( 'help' ) ),
|
|
repo_owner_username=to_unicode( repo.get( 'repo_owner_username' ) ),
|
|
repo_name=to_unicode( repo.get( 'name' ) ),
|
|
repo_id=repo.get( 'id' ) )
|
|
tools_indexed += 1
|
|
print tools_indexed, 'tools (', tool.get( 'id' ), ')'
|
|
|
|
repos_indexed += 1
|
|
print repos_indexed, 'repos (', repo.get( 'id' ), ')'
|
|
|
|
tool_index_writer.commit()
|
|
repo_index_writer.commit()
|
|
|
|
print "TOTAL repos indexed: ", repos_indexed
|
|
print "TOTAL tools indexed: ", tools_indexed
|
|
|
|
|
|
def get_repos( sa_session, path_to_repositories ):
|
|
"""
|
|
Load repos from DB and included tools from .xml configs.
|
|
"""
|
|
results = []
|
|
for repo in sa_session.query( model.Repository ).filter_by( deleted=False ).filter_by( deprecated=False ).filter( model.Repository.type != 'tool_dependency_definition' ):
|
|
|
|
repo_id = repo.id
|
|
name = repo.name
|
|
description = repo.description
|
|
long_description = repo.long_description
|
|
homepage_url = repo.homepage_url
|
|
remote_repository_url = repo.remote_repository_url
|
|
|
|
times_downloaded = repo.times_downloaded
|
|
if not isinstance( times_downloaded, ( int, long ) ):
|
|
times_downloaded = 0
|
|
|
|
repo_owner_username = ''
|
|
if repo.user_id is not None:
|
|
user = sa_session.query( model.User ).filter( model.User.id == repo.user_id ).one()
|
|
repo_owner_username = user.username
|
|
|
|
approved = 'no'
|
|
for review in repo.reviews:
|
|
if review.approved == 'yes':
|
|
approved = 'yes'
|
|
break
|
|
|
|
# Format the time since last update to be nicely readable.
|
|
last_updated = pretty_print_time_interval( repo.update_time )
|
|
full_last_updated = repo.update_time.strftime( "%Y-%m-%d %I:%M %p" )
|
|
|
|
# Parse all the tools within repo for separate index.
|
|
tools_list = []
|
|
path = os.path.join( path_to_repositories, *model.directory_hash_id( repo.id ))
|
|
path = os.path.join( path, "repo_%d" % repo.id )
|
|
if os.path.exists(path):
|
|
tools_list.extend( load_one_dir( path ) )
|
|
for root, dirs, files in os.walk( path ):
|
|
if '.hg' in dirs:
|
|
dirs.remove('.hg')
|
|
for dirname in dirs:
|
|
tools_in_dir = load_one_dir( os.path.join( root, dirname ) )
|
|
tools_list.extend( tools_in_dir )
|
|
|
|
results.append(dict( id=repo_id,
|
|
name=name,
|
|
description=description,
|
|
long_description=long_description,
|
|
homepage_url=homepage_url,
|
|
remote_repository_url=remote_repository_url,
|
|
repo_owner_username=repo_owner_username,
|
|
times_downloaded=times_downloaded,
|
|
approved=approved,
|
|
last_updated=last_updated,
|
|
full_last_updated=full_last_updated,
|
|
tools_list=tools_list ) )
|
|
return results
|
|
|
|
|
|
def load_one_dir( path ):
|
|
tools_in_dir = []
|
|
tool_elems = load_tool_elements_from_path( path )
|
|
if tool_elems:
|
|
for elem in tool_elems:
|
|
root = elem[1].getroot()
|
|
if root.tag == 'tool':
|
|
tool = {}
|
|
if root.find( 'help' ) is not None:
|
|
tool.update( dict( help=root.find( 'help' ).text ) )
|
|
if root.find( 'description' ) is not None:
|
|
tool.update( dict( description=root.find( 'description' ).text ) )
|
|
tool.update( dict( id=root.attrib.get( 'id' ),
|
|
name=root.attrib.get( 'name' ),
|
|
version=root.attrib.get( 'version' ) ) )
|
|
tools_in_dir.append( tool )
|
|
return tools_in_dir
|
|
|
|
|
|
def get_sa_session_and_needed_config_settings( path_to_tool_shed_config ):
|
|
conf_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } )
|
|
conf_parser.read( path_to_tool_shed_config )
|
|
kwds = dict()
|
|
for key, value in conf_parser.items( "app:main" ):
|
|
kwds[ key ] = value
|
|
config_settings = config.Configuration( **kwds )
|
|
db_con = config_settings.database_connection
|
|
if not db_con:
|
|
db_con = "sqlite:///%s?isolation_level=IMMEDIATE" % config_settings.database
|
|
model = galaxy.webapps.tool_shed.model.mapping.init( config_settings.file_path, db_con, engine_options={}, create_tables=False )
|
|
return model.context.current, config_settings
|
|
|
|
if __name__ == "__main__":
|
|
parser = OptionParser()
|
|
parser.add_option("-c", "--config", dest="path_to_tool_shed_config", default="config/tool_shed.ini", help="specify tool_shed.ini location")
|
|
(options, args) = parser.parse_args()
|
|
path_to_tool_shed_config = options.path_to_tool_shed_config
|
|
sa_session, config_settings = get_sa_session_and_needed_config_settings( path_to_tool_shed_config )
|
|
whoosh_index_dir = config_settings.get( 'whoosh_index_dir', None )
|
|
path_to_repositories = config_settings.get( 'file_path', 'database/community_files' )
|
|
build_index( sa_session, whoosh_index_dir, path_to_repositories )
|