opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Eliminate the use of the Galaxy web transaction object in Tool Shed utility functions related to importing a repository capsule into a Tool Shed and move som hg related functions from the commit_util module to the hg_util module.

Browse Source
This commit is contained in:
Greg Von Kuster
2014-06-18 15:28:20 -04:00
parent 3ac10858ca
commit 4297b7054a
15 changed files with 619 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py
+1 -1
View File
@@ -419,7 +419,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
try:
invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans.app, repository_id )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans,
message = tool_util.generate_message_for_invalid_tools( trans.app,
invalid_file_tups,
repository,
None,
lib/galaxy/webapps/tool_shed/api/repositories.py
+35 -17
View File
@@ -250,8 +250,8 @@ class RepositoriesController( BaseAPIController ):
return {}
capsule_dict[ 'tar_archive' ] = tar_archive
capsule_dict[ 'capsule_file_name' ] = capsule_file_name
capsule_dict = import_util.extract_capsule_files( trans, **capsule_dict )
capsule_dict = import_util.validate_capsule( trans, **capsule_dict )
capsule_dict = import_util.extract_capsule_files( **capsule_dict )
capsule_dict = import_util.validate_capsule( **capsule_dict )
status = capsule_dict.get( 'status', 'error' )
if status == 'error':
log.debug( 'The capsule contents are invalid and cannot be imported:<br/>%s' % \
@@ -268,7 +268,10 @@ class RepositoriesController( BaseAPIController ):
# The manifest.xml file has already been validated, so no error_message should be returned here.
repository_info_dicts, error_message = import_util.get_repository_info_from_manifest( manifest_file_path )
# Determine the status for each exported repository archive contained within the capsule.
repository_status_info_dicts = import_util.get_repository_status_from_tool_shed( trans, repository_info_dicts )
repository_status_info_dicts = import_util.get_repository_status_from_tool_shed( trans.app,
trans.user,
trans.user_is_admin(),
repository_info_dicts )
# Generate a list of repository name / import results message tuples for display after the capsule is imported.
import_results_tups = []
# Only create repositories that do not yet exist and that the current user is authorized to create. The
@@ -277,11 +280,12 @@ class RepositoriesController( BaseAPIController ):
# Add the capsule_file_name and encoded_file_path to the repository_status_info_dict.
repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name
repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path
import_results_tups = \
repository_maintenance_util.create_repository_and_import_archive( trans,
repository_status_info_dict,
import_results_tups )
import_util.check_status_and_reset_downloadable( trans, import_results_tups )
import_results_tups = import_util.create_repository_and_import_archive( trans.app,
trans.request.host,
trans.user,
repository_status_info_dict,
import_results_tups )
import_util.check_status_and_reset_downloadable( trans.app, import_results_tups )
basic_util.remove_dir( file_path )
# NOTE: the order of installation is defined in import_results_tups, but order will be lost
# when transferred to return_dict.
@@ -413,15 +417,22 @@ class RepositoriesController( BaseAPIController ):
log.debug( "Resetting metadata on repository %s" % str( repository.name ) )
repository_id = trans.security.encode_id( repository.id )
try:
invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, repository_id )
invalid_file_tups, metadata_dict = \
metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, repository_id )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False )
message = tool_util.generate_message_for_invalid_tools( trans.app,
invalid_file_tups,
repository,
None,
as_html=False )
results[ 'unsuccessful_count' ] += 1
else:
message = "Successfully reset metadata on repository %s owned by %s" % ( str( repository.name ), str( repository.user.username ) )
message = "Successfully reset metadata on repository %s owned by %s" % \
( str( repository.name ), str( repository.user.username ) )
results[ 'successful_count' ] += 1
except Exception, e:
message = "Error resetting metadata on repository %s owned by %s: %s" % ( str( repository.name ), str( repository.user.username ), str( e ) )
message = "Error resetting metadata on repository %s owned by %s: %s" % \
( str( repository.name ), str( repository.user.username ), str( e ) )
results[ 'unsuccessful_count' ] += 1
status = '%s : %s' % ( str( repository.name ), message )
results[ 'repository_status' ].append( status )
@@ -486,14 +497,21 @@ class RepositoriesController( BaseAPIController ):
results = dict( start_time=start_time,
repository_status=[] )
try:
invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans,
trans.security.encode_id( repository.id ) )
invalid_file_tups, metadata_dict = \
metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans,
trans.security.encode_id( repository.id ) )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False )
message = tool_util.generate_message_for_invalid_tools( trans.app,
invalid_file_tups,
repository,
None,
as_html=False )
else:
message = "Successfully reset metadata on repository %s owned by %s" % ( str( repository.name ), str( repository.user.username ) )
message = "Successfully reset metadata on repository %s owned by %s" % \
( str( repository.name ), str( repository.user.username ) )
except Exception, e:
message = "Error resetting metadata on repository %s owned by %s: %s" % ( str( repository.name ), str( repository.user.username ), str( e ) )
message = "Error resetting metadata on repository %s owned by %s: %s" % \
( str( repository.name ), str( repository.user.username ), str( e ) )
status = '%s : %s' % ( str( repository.name ), message )
results[ 'repository_status' ].append( status )
return results
lib/galaxy/webapps/tool_shed/controllers/admin.py
+5 -2
View File
@@ -315,11 +315,14 @@ class AdminController( BaseUIController, Admin ):
def manage_role_associations( self, trans, **kwd ):
"""Manage users, groups and repositories associated with a role."""
role_id = kwd.get( 'id', None )
role = repository_maintenance_util.get_role_by_id( trans, role_id )
role = repository_maintenance_util.get_role_by_id( trans.app, role_id )
# We currently only have a single role associated with a repository, the repository admin role.
repository_role_association = role.repositories[ 0 ]
repository = repository_role_association.repository
associations_dict = repository_maintenance_util.handle_role_associations( trans, role, repository, **kwd )
associations_dict = repository_maintenance_util.handle_role_associations( trans.app,
role,
repository,
**kwd )
in_users = associations_dict.get( 'in_users', [] )
out_users = associations_dict.get( 'out_users', [] )
in_groups = associations_dict.get( 'in_groups', [] )
lib/galaxy/webapps/tool_shed/controllers/hg.py
+1 -1
View File
@@ -45,7 +45,7 @@ class HgController( BaseUIController ):
repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) )
update_repository( repo, ctx_rev=None )
# Set metadata using the repository files on disk.
error_message, status = set_repository_metadata( trans, repository )
error_message, status = set_repository_metadata( trans.app, trans.request.host, trans.user, repository )
if status == 'ok' and error_message:
log.debug( "Successfully reset metadata on repository %s owned by %s, but encountered problem: %s" % \
( str( repository.name ), str( repository.user.username ), error_message ) )
lib/galaxy/webapps/tool_shed/controllers/repository.py
+42 -23
View File
@@ -41,7 +41,6 @@ import tool_shed.repository_types.util as rt_util
from galaxy import eggs
eggs.require( 'mercurial' )
from mercurial import commands
from mercurial import mdiff
from mercurial import patch
@@ -849,7 +848,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
operation = kwd[ 'operation' ].lower()
if operation == "preview_tools_in_changeset":
repository = suc.get_repository_in_tool_shed( trans.app, repository_id )
repository_metadata = metadata_util.get_latest_repository_metadata( trans, repository.id, downloadable=True )
repository_metadata = metadata_util.get_latest_repository_metadata( trans.app, repository.id, downloadable=True )
latest_installable_changeset_revision = repository_metadata.changeset_revision
return trans.response.send_redirect( web.url_for( controller='repository',
action='preview_tools_in_changeset',
@@ -1017,7 +1016,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
if error:
status = 'error'
else:
repository, message = repository_maintenance_util.create_repository( trans,
repository, message = repository_maintenance_util.create_repository( trans.app,
name,
repository_type,
description,
@@ -1974,7 +1973,10 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
# The manifest.xml file has already been validated, so no error_message should be returned here.
repository_info_dicts, error_message = import_util.get_repository_info_from_manifest( manifest_file_path )
# Determine the status for each exported repository archive contained within the capsule.
repository_status_info_dicts = import_util.get_repository_status_from_tool_shed( trans, repository_info_dicts )
repository_status_info_dicts = import_util.get_repository_status_from_tool_shed( trans.app,
trans.user,
trans.user_is_admin(),
repository_info_dicts )
if 'import_capsule_button' in kwd:
# Generate a list of repository name / import results message tuples for display after the capsule is imported.
import_results_tups = []
@@ -1984,11 +1986,12 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
# Add the capsule_file_name and encoded_file_path to the repository_status_info_dict.
repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name
repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path
import_results_tups = \
repository_maintenance_util.create_repository_and_import_archive( trans,
repository_status_info_dict,
import_results_tups )
import_util.check_status_and_reset_downloadable( trans, import_results_tups )
import_results_tups = import_util.create_repository_and_import_archive( trans.app,
trans.request.host,
trans.user,
repository_status_info_dict,
import_results_tups )
import_util.check_status_and_reset_downloadable( trans.app, import_results_tups )
basic_util.remove_dir( file_path )
return trans.fill_template( '/webapps/tool_shed/repository/import_capsule_results.mako',
export_info_dict=export_info_dict,
@@ -2099,7 +2102,12 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
tool,
[] )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, {}, as_html=True, displaying_invalid_tool=True )
message = tool_util.generate_message_for_invalid_tools( trans.app,
invalid_file_tups,
repository,
{},
as_html=True,
displaying_invalid_tool=True )
elif error_message:
message = error_message
try:
@@ -2459,7 +2467,10 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
if repository_metadata:
metadata = repository_metadata.metadata
role = repository.admin_role
associations_dict = repository_maintenance_util.handle_role_associations( trans, role, repository, **kwd )
associations_dict = repository_maintenance_util.handle_role_associations( trans.app,
role,
repository,
**kwd )
in_users = associations_dict.get( 'in_users', [] )
out_users = associations_dict.get( 'out_users', [] )
in_groups = associations_dict.get( 'in_groups', [] )
@@ -2693,7 +2704,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, id, **kwd )
if invalid_file_tups:
repository = suc.get_repository_in_tool_shed( trans.app, id )
message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict )
message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, repository, metadata_dict )
status = 'error'
else:
message = "All repository metadata has been reset. "
@@ -2733,7 +2744,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
tip = repository.tip( trans.app )
for selected_file in selected_files_to_delete:
try:
commands.remove( repo.ui, repo, selected_file, force=True )
hg_util.remove_file( repo.ui, repo, selected_file, force=True )
except Exception, e:
log.debug( "Error removing the following file using the mercurial API:\n %s" % str( selected_file ) )
log.debug( "The error was: %s" % str( e ))
@@ -2759,19 +2770,27 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
# Commit the change set.
if not commit_message:
commit_message = 'Deleted selected files'
commands.commit( repo.ui, repo, repo_dir, user=trans.user.username, message=commit_message )
suc.handle_email_alerts( trans, repository )
hg_util.commit_changeset( repo.ui,
repo,
full_path_to_changeset=repo_dir,
username=trans.user.username,
message=commit_message )
suc.handle_email_alerts( trans.app, trans.request.host, repository )
# Update the repository files for browsing.
hg_util.update_repository( repo )
# Get the new repository tip.
if tip == repository.tip( trans.app ):
message += 'No changes to repository. '
kwd[ 'message' ] = message
else:
message += 'The selected files were deleted from the repository. '
kwd[ 'message' ] = message
metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, **kwd )
status, error_message = metadata_util.set_repository_metadata_due_to_new_tip( trans.app,
trans.request.host,
trans.user,
repository,
**kwd )
if error_message:
message = error_message
else:
message += 'The selected files were deleted from the repository. '
else:
message = "Select at least 1 file to delete from the repository before clicking <b>Delete selected files</b>."
status = "error"
@@ -3061,13 +3080,13 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
status = kwd.get( 'status', 'done' )
url = kwd.get( 'url', '' )
if 'upload_capsule_button' in kwd:
capsule_dict = import_util.upload_capsule( trans, **kwd )
capsule_dict = import_util.upload_capsule( **kwd )
status = capsule_dict.get( 'status', 'error' )
if status == 'error':
message = capsule_dict.get( 'error_message', '' )
else:
capsule_dict = import_util.extract_capsule_files( trans, **capsule_dict )
capsule_dict = import_util.validate_capsule( trans, **capsule_dict )
capsule_dict = import_util.extract_capsule_files( **capsule_dict )
capsule_dict = import_util.validate_capsule( **capsule_dict )
status = capsule_dict.get( 'status', 'error' )
if status == 'ok':
return trans.response.send_redirect( web.url_for( controller='repository',
lib/galaxy/webapps/tool_shed/controllers/upload.py
+37 -14
View File
@@ -167,7 +167,7 @@ class UploadController( BaseUIController ):
# Inspect the contents of the file to see if changeset_revision values are missing and if so,
# set them appropriately.
altered, root_elem, error_message = \
commit_util.handle_repository_dependencies_definition( trans,
commit_util.handle_repository_dependencies_definition( trans.app,
uploaded_file_name,
unpopulate=False )
if error_message:
@@ -184,7 +184,7 @@ class UploadController( BaseUIController ):
# Inspect the contents of the file to see if changeset_revision values are
# missing and if so, set them appropriately.
altered, root_elem, error_message = \
commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
commit_util.handle_tool_dependencies_definition( trans.app, uploaded_file_name )
if error_message:
ok = False
message = error_message
@@ -200,15 +200,19 @@ class UploadController( BaseUIController ):
if ok:
# See if any admin users have chosen to receive email alerts when a repository is updated.
# If so, check every uploaded file to ensure content is appropriate.
check_contents = commit_util.check_file_contents_for_email_alerts( trans )
check_contents = commit_util.check_file_contents_for_email_alerts( trans.app )
if check_contents and os.path.isfile( full_path ):
content_alert_str = commit_util.check_file_content_for_html_and_images( full_path )
else:
content_alert_str = ''
commands.add( repo.ui, repo, full_path )
hg_util.add_changeset( repo.ui, repo, full_path )
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode( 'ascii', 'replace' )
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
hg_util.commit_changeset( repo.ui,
repo,
full_path_to_changeset=full_path,
username=trans.user.username,
message=commit_message )
if full_path.endswith( 'tool_data_table_conf.xml.sample' ):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
# by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables
@@ -218,7 +222,8 @@ class UploadController( BaseUIController ):
message = '%s<br/>%s' % ( message, error_message )
# See if the content of the change set was valid.
admin_only = len( repository.downloadable_revisions ) != 1
suc.handle_email_alerts( trans,
suc.handle_email_alerts( trans.app,
trans.request.host,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
@@ -253,8 +258,15 @@ class UploadController( BaseUIController ):
( len( files_to_remove ), upload_point )
else:
message += " %d files were removed from the repository root. " % len( files_to_remove )
status, error_message = metadata_util.set_repository_metadata_due_to_new_tip( trans.app,
trans.request.host,
trans.user,
repository,
content_alert_str=content_alert_str,
**kwd )
if error_message:
message = error_message
kwd[ 'message' ] = message
metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd )
if repository.metadata_revisions:
# A repository's metadata revisions are order descending by update_time, so the zeroth revision
# will be the tip just after an upload.
@@ -355,7 +367,7 @@ class UploadController( BaseUIController ):
# Inspect the contents of the file to see if changeset_revision values are missing and
# if so, set them appropriately.
altered, root_elem, error_message = \
commit_util.handle_repository_dependencies_definition( trans,
commit_util.handle_repository_dependencies_definition( trans.app,
uploaded_file_name,
unpopulate=False )
if error_message:
@@ -365,7 +377,7 @@ class UploadController( BaseUIController ):
shutil.move( tmp_filename, uploaded_file_name )
elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans.app, uploaded_file_name )
if error_message:
return False, error_message, [], '', [], []
if altered:
@@ -382,8 +394,17 @@ class UploadController( BaseUIController ):
os.remove( repo_path )
shutil.move( os.path.join( uploaded_directory, relative_path ), repo_path )
filenames_in_archive.append( relative_path )
return commit_util.handle_directory_changes( trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar,
new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed )
return commit_util.handle_directory_changes( trans.app,
trans.request.host,
trans.user.username,
repository,
full_path,
filenames_in_archive,
remove_repo_files_not_in_tar,
new_repo_alert,
commit_message,
undesirable_dirs_removed,
undesirable_files_removed )
def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ):
# Upload a tar archive of files.
@@ -422,7 +443,7 @@ class UploadController( BaseUIController ):
uploaded_file_name = os.path.join( full_path, filename )
if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans,
altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans.app,
uploaded_file_name,
unpopulate=False )
if error_message:
@@ -432,13 +453,15 @@ class UploadController( BaseUIController ):
shutil.move( tmp_filename, uploaded_file_name )
elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans.app, uploaded_file_name )
if error_message:
return False, error_message, [], '', [], []
if altered:
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, uploaded_file_name )
return commit_util.handle_directory_changes( trans,
return commit_util.handle_directory_changes( trans.app,
trans.request.host,
trans.user.username,
repository,
full_path,
filenames_in_archive,
lib/galaxy/webapps/tool_shed/framework/middleware/hg.py
+4 -5
View File
@@ -1,6 +1,7 @@
"""Middle-ware for handling hg authentication for users pushing change sets to local repositories."""
import os
import json
import logging
import os
import sqlalchemy
import sys
import tempfile
@@ -9,11 +10,9 @@ from paste.httpheaders import AUTH_TYPE
from paste.httpheaders import REMOTE_USER
from galaxy.util import asbool
from galaxy.util import json
from galaxy.webapps.tool_shed import model
from galaxy.util.hash_util import new_secure_hash
import tool_shed.util.shed_util_common as suc
from tool_shed.util import commit_util
from tool_shed.util import hg_util
import tool_shed.repository_types.util as rt_util
from galaxy import eggs
@@ -104,7 +103,7 @@ class Hg( object ):
fh.write( chunk )
fh.close()
fh = open( tmp_filename, 'rb' )
changeset_groups = json.from_json_string( commit_util.bundle_to_json( fh ) )
changeset_groups = json.loads( hg_util.bundle_to_json( fh ) )
fh.close()
try:
os.unlink( tmp_filename )
lib/tool_shed/util/commit_util.py
+61 -122
View File
@@ -1,14 +1,11 @@
import cStringIO
import gzip
import json
import logging
import os
import pkg_resources
import shutil
import struct
import tempfile
from galaxy.datatypes import checkers
from galaxy.util import asbool
from galaxy.util import json
from galaxy.util.odict import odict
from galaxy.web import url_for
import tool_shed.util.shed_util_common as suc
@@ -18,24 +15,11 @@ from tool_shed.util import tool_util
from tool_shed.util import xml_util
import tool_shed.repository_types.util as rt_util
from galaxy import eggs
eggs.require( 'mercurial' )
from mercurial import commands
from mercurial.changegroup import readbundle
from mercurial.changegroup import readexactly
log = logging.getLogger( __name__ )
UNDESIRABLE_DIRS = [ '.hg', '.svn', '.git', '.cvs' ]
UNDESIRABLE_FILES = [ '.hg_archival.txt', 'hgrc', '.DS_Store' ]
def bundle_to_json( fh ):
"""Convert the received HG10xx data stream (a mercurial 1.0 bundle created using hg push from the command line) to a json object."""
# See http://www.wstein.org/home/wstein/www/home/was/patches/hg_json
hg_unbundle10_obj = readbundle( fh, None )
groups = [ group for group in unpack_groups( hg_unbundle10_obj ) ]
return json.to_json_string( groups, indent=4 )
def check_archive( repository, archive ):
for member in archive.getmembers():
# Allow regular files and directories only
@@ -62,15 +46,16 @@ def check_archive( repository, archive ):
return False, message
return True, ''
def check_file_contents_for_email_alerts( trans ):
def check_file_contents_for_email_alerts( app ):
"""
See if any admin users have chosen to receive email alerts when a repository is updated. If so, the file contents of the update must be
checked for inappropriate content.
"""
admin_users = trans.app.config.get( "admin_users", "" ).split( "," )
for repository in trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.email_alerts != None ):
email_alerts = json.from_json_string( repository.email_alerts )
sa_session = app.model.context.current
admin_users = app.config.get( "admin_users", "" ).split( "," )
for repository in sa_session.query( app.model.Repository ) \
.filter( app.model.Repository.table.c.email_alerts != None ):
email_alerts = json.loads( repository.email_alerts )
for user_email in email_alerts:
if user_email in admin_users:
return True
@@ -140,14 +125,14 @@ def handle_bz2( repository, uploaded_file_name ):
bzipped_file.close()
shutil.move( uncompressed, uploaded_file_name )
def handle_complex_repository_dependency_elem( trans, elem, sub_elem_index, sub_elem, sub_elem_altered, altered, unpopulate=False ):
def handle_complex_repository_dependency_elem( app, elem, sub_elem_index, sub_elem, sub_elem_altered, altered, unpopulate=False ):
"""
Populate or unpopulate the toolshed and changeset_revision attributes of a <repository> tag that defines
a complex repository dependency.
"""
# The received sub_elem looks something like the following:
# <repository name="package_eigen_2_0" owner="test" prior_installation_required="True" />
revised, repository_elem, error_message = handle_repository_dependency_elem( trans, sub_elem, unpopulate=unpopulate )
revised, repository_elem, error_message = handle_repository_dependency_elem( app, sub_elem, unpopulate=unpopulate )
if error_message:
error_message = 'The tool_dependencies.xml file contains an invalid <repository> tag. %s' % error_message
if revised:
@@ -157,13 +142,13 @@ def handle_complex_repository_dependency_elem( trans, elem, sub_elem_index, sub_
altered = True
return altered, sub_elem_altered, elem, error_message
def handle_directory_changes( trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert,
commit_message, undesirable_dirs_removed, undesirable_files_removed ):
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
def handle_directory_changes( app, host, username, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar,
new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed ):
repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False )
content_alert_str = ''
files_to_remove = []
filenames_in_archive = [ os.path.join( full_path, name ) for name in filenames_in_archive ]
if remove_repo_files_not_in_tar and not repository.is_new( trans.app ):
if remove_repo_files_not_in_tar and not repository.is_new( app ):
# We have a repository that is not new (it contains files), so discover those files that are in the
# repository, but not in the uploaded archive.
for root, dirs, files in os.walk( full_path ):
@@ -181,9 +166,10 @@ def handle_directory_changes( trans, repository, full_path, filenames_in_archive
if full_name not in filenames_in_archive:
files_to_remove.append( full_name )
for repo_file in files_to_remove:
# Remove files in the repository (relative to the upload point) that are not in the uploaded archive.
# Remove files in the repository (relative to the upload point) that are not in
# the uploaded archive.
try:
commands.remove( repo.ui, repo, repo_file, force=True )
hg_util.remove_file( repo.ui, repo, repo_file, force=True )
except Exception, e:
log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str( e ))
relative_selected_file = repo_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' )
@@ -204,23 +190,33 @@ def handle_directory_changes( trans, repository, full_path, filenames_in_archive
except OSError, e:
# The directory is not empty.
pass
# See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure
# content is appropriate.
check_contents = check_file_contents_for_email_alerts( trans )
# See if any admin users have chosen to receive email alerts when a repository is updated.
# If so, check every uploaded file to ensure content is appropriate.
check_contents = check_file_contents_for_email_alerts( app )
for filename_in_archive in filenames_in_archive:
# Check file content to ensure it is appropriate.
if check_contents and os.path.isfile( filename_in_archive ):
content_alert_str += check_file_content_for_html_and_images( filename_in_archive )
commands.add( repo.ui, repo, filename_in_archive )
hg_util.add_changeset( repo.ui, repo, filename_in_archive )
if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary.
error, message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive )
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
# by parsing the file and adding new entries to the in-memory app.tool_data_tables
# dictionary.
error, message = tool_util.handle_sample_tool_data_table_conf_file( app, filename_in_archive )
if error:
return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
hg_util.commit_changeset( repo.ui,
repo,
full_path_to_changeset=full_path,
username=username,
message=commit_message )
admin_only = len( repository.downloadable_revisions ) != 1
suc.handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only )
suc.handle_email_alerts( app,
host,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only )
return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
def handle_missing_repository_attribute( elem ):
@@ -253,7 +249,7 @@ def handle_gzip( repository, uploaded_file_name ):
gzipped_file.close()
shutil.move( uncompressed, uploaded_file_name )
def handle_repository_dependencies_definition( trans, repository_dependencies_config, unpopulate=False ):
def handle_repository_dependencies_definition( app, repository_dependencies_config, unpopulate=False ):
"""
Populate or unpopulate the toolshed and changeset_revision attributes of a <repository> tag. Populating will occur when a
dependency definition file is being uploaded to the repository, while depopulating will occur when the repository is being
@@ -269,7 +265,7 @@ def handle_repository_dependencies_definition( trans, repository_dependencies_co
for index, elem in enumerate( root ):
if elem.tag == 'repository':
# <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" />
revised, elem, error_message = handle_repository_dependency_elem( trans, elem, unpopulate=unpopulate )
revised, elem, error_message = handle_repository_dependency_elem( app, elem, unpopulate=unpopulate )
if error_message:
error_message = 'The repository_dependencies.xml file contains an invalid <repository> tag. %s' % error_message
return False, None, error_message
@@ -280,7 +276,7 @@ def handle_repository_dependencies_definition( trans, repository_dependencies_co
return altered, root, error_message
return False, None, error_message
def handle_repository_dependency_elem( trans, elem, unpopulate=False ):
def handle_repository_dependency_elem( app, elem, unpopulate=False ):
"""Populate or unpopulate repository tags."""
# <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" />
# <repository changeset_revision="xxx" name="package_xorg_macros_1_17_1" owner="test" toolshed="yyy">
@@ -342,10 +338,10 @@ def handle_repository_dependency_elem( trans, elem, unpopulate=False ):
# Populate the changeset_revision attribute with the latest installable metadata revision for the defined repository.
# We use the latest installable revision instead of the latest metadata revision to ensure that the contents of the
# revision are valid.
repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
repository = suc.get_repository_by_name_and_owner( app, name, owner )
if repository:
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo )
repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False )
lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( app, repository, repo )
if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH:
elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision
revised = True
@@ -356,7 +352,7 @@ def handle_repository_dependency_elem( trans, elem, unpopulate=False ):
error_message = 'Unable to locate repository with name %s and owner %s. ' % ( str( name ), str( owner ) )
return revised, elem, error_message
def handle_repository_dependency_sub_elem( trans, package_altered, altered, actions_elem, action_index, action_elem, unpopulate=False ):
def handle_repository_dependency_sub_elem( app, package_altered, altered, actions_elem, action_index, action_elem, unpopulate=False ):
"""
Populate or unpopulate the toolshed and changeset_revision attributes for each of the following tag sets.
<action type="set_environment_for_install">
@@ -367,7 +363,7 @@ def handle_repository_dependency_sub_elem( trans, package_altered, altered, acti
for repo_index, repo_elem in enumerate( action_elem ):
# Make sure to skip comments and tags that are not <repository>.
if repo_elem.tag == 'repository':
revised, repository_elem, message = handle_repository_dependency_elem( trans, repo_elem, unpopulate=unpopulate )
revised, repository_elem, message = handle_repository_dependency_elem( app, repo_elem, unpopulate=unpopulate )
if message:
error_message += 'The tool_dependencies.xml file contains an invalid <repository> tag. %s' % message
if revised:
@@ -379,7 +375,7 @@ def handle_repository_dependency_sub_elem( trans, package_altered, altered, acti
actions_elem[ action_index ] = action_elem
return package_altered, altered, actions_elem, error_message
def handle_tool_dependencies_definition( trans, tool_dependencies_config, unpopulate=False ):
def handle_tool_dependencies_definition( app, tool_dependencies_config, unpopulate=False ):
"""
Populate or unpopulate the tooshed and changeset_revision attributes of each <repository>
tag defined within a tool_dependencies.xml file.
@@ -401,7 +397,7 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config, unpopu
if package_elem.tag == 'repository':
# We have a complex repository dependency.
altered, package_altered, root_elem, message = \
handle_complex_repository_dependency_elem( trans,
handle_complex_repository_dependency_elem( app,
root_elem,
package_index,
package_elem,
@@ -440,7 +436,7 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config, unpopu
if last_actions_elem_package_elem.tag == 'repository':
# We have a complex repository dependency.
altered, last_actions_package_altered, last_actions_elem, message = \
handle_complex_repository_dependency_elem( trans,
handle_complex_repository_dependency_elem( app,
last_actions_elem,
last_actions_elem_package_index,
last_actions_elem_package_elem,
@@ -456,7 +452,7 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config, unpopu
# Inspect the sub elements of last_actions_elem to locate all <repository> tags and
# populate them with toolshed and changeset_revision attributes if necessary.
last_actions_package_altered, altered, last_actions_elem, message = \
handle_repository_dependency_sub_elem( trans,
handle_repository_dependency_sub_elem( app,
last_actions_package_altered,
altered,
actions_group_elem,
@@ -468,23 +464,25 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config, unpopu
elif actions_elem.tag == 'actions':
# We are not in an <actions_group> tag set, so we must be in an <actions> tag set.
for action_index, action_elem in enumerate( actions_elem ):
# Inspect the sub elements of last_actions_elem to locate all <repository> tags and populate them with
# toolshed and changeset_revision attributes if necessary.
package_altered, altered, actions_elem, message = handle_repository_dependency_sub_elem( trans,
package_altered,
altered,
actions_elem,
action_index,
action_elem,
unpopulate=unpopulate )
# Inspect the sub elements of last_actions_elem to locate all <repository> tags
# and populate them with toolshed and changeset_revision attributes if necessary.
package_altered, altered, actions_elem, message = \
handle_repository_dependency_sub_elem( app,
package_altered,
altered,
actions_elem,
action_index,
action_elem,
unpopulate=unpopulate )
if message:
error_message += message
else:
package_name = root_elem.get( 'name', '' )
package_version = root_elem.get( 'version', '' )
error_message += 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) )
error_message += 'the recipe for installing the package is missing either an &lt;actions&gt; tag set or an &lt;actions_group&gt; '
error_message += 'tag set.'
error_message += 'Version %s of the %s package cannot be installed because ' % \
( str( package_version ), str( package_name ) )
error_message += 'the recipe for installing the package is missing either an '
error_message += '&lt;actions&gt; tag set or an &lt;actions_group&gt; tag set.'
if package_altered:
package_elem[ actions_index ] = actions_elem
if package_altered:
@@ -534,62 +532,3 @@ def uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip=F
if isbz2:
handle_bz2( repository, uploaded_file_name )
return uploaded_file_filename.rstrip( '.bz2' )
def unpack_chunks( hg_unbundle10_obj ):
"""
This method provides a generator of parsed chunks of a "group" in a mercurial unbundle10 object which
is created when a changeset that is pushed to a Tool Shed repository using hg push from the command line
is read using readbundle.
"""
while True:
length, = struct.unpack( '>l', readexactly( hg_unbundle10_obj, 4 ) )
if length <= 4:
# We found a "null chunk", which ends the group.
break
if length < 84:
raise Exception( "negative data length" )
node, p1, p2, cs = struct.unpack( '20s20s20s20s', readexactly( hg_unbundle10_obj, 80 ) )
yield { 'node': node.encode( 'hex' ),
'p1': p1.encode( 'hex' ),
'p2': p2.encode( 'hex' ),
'cs': cs.encode( 'hex' ),
'data': [ patch for patch in unpack_patches( hg_unbundle10_obj, length - 84 ) ] }
def unpack_groups( hg_unbundle10_obj ):
"""
This method provides a generator of parsed groups from a mercurial unbundle10 object which is
created when a changeset that is pushed to a Tool Shed repository using hg push from the command
line is read using readbundle.
"""
# Process the changelog group.
yield [ chunk for chunk in unpack_chunks( hg_unbundle10_obj ) ]
# Process the manifest group.
yield [ chunk for chunk in unpack_chunks( hg_unbundle10_obj ) ]
while True:
length, = struct.unpack( '>l', readexactly( hg_unbundle10_obj, 4 ) )
if length <= 4:
# We found a "null meta chunk", which ends the changegroup.
break
filename = readexactly( hg_unbundle10_obj, length-4 ).encode( 'string_escape' )
# Process the file group.
yield ( filename, [ chunk for chunk in unpack_chunks( hg_unbundle10_obj ) ] )
def unpack_patches( hg_unbundle10_obj, remaining ):
"""
This method provides a generator of patches from the data field in a chunk. As there is no delimiter
for this data field, a length argument is required.
"""
while remaining >= 12:
start, end, blocklen = struct.unpack( '>lll', readexactly( hg_unbundle10_obj, 12 ) )
remaining -= 12
if blocklen > remaining:
raise Exception( "unexpected end of patch stream" )
block = readexactly( hg_unbundle10_obj, blocklen )
remaining -= blocklen
yield { 'start': start,
'end': end,
'blocklen': blocklen,
'block': block.encode( 'string_escape' ) }
if remaining > 0:
print remaining
raise Exception( "unexpected end of patch stream" )
lib/tool_shed/util/export_util.py
+2 -2
View File
@@ -151,7 +151,7 @@ def generate_repository_archive( trans, work_dir, tool_shed_url, repository, cha
if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
# Eliminate the toolshed, and changeset_revision attributes from all <repository> tags.
altered, root_elem, error_message = \
commit_util.handle_repository_dependencies_definition( trans, full_path, unpopulate=True )
commit_util.handle_repository_dependencies_definition( trans.app, full_path, unpopulate=True )
if error_message:
return None, error_message
if altered:
@@ -160,7 +160,7 @@ def generate_repository_archive( trans, work_dir, tool_shed_url, repository, cha
elif name == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
# Eliminate the toolshed, and changeset_revision attributes from all <repository> tags.
altered, root_elem, error_message = \
commit_util.handle_tool_dependencies_definition( trans, full_path, unpopulate=True )
commit_util.handle_tool_dependencies_definition( trans.app, full_path, unpopulate=True )
if error_message:
return None, error_message
if altered:
lib/tool_shed/util/hg_util.py
+82
View File
@@ -1,5 +1,7 @@
import json
import logging
import os
import struct
from datetime import datetime
from time import gmtime
@@ -14,6 +16,8 @@ from mercurial import cmdutil
from mercurial import commands
from mercurial import hg
from mercurial import ui
from mercurial.changegroup import readbundle
from mercurial.changegroup import readexactly
from tool_shed.util import basic_util
@@ -21,6 +25,19 @@ log = logging.getLogger( __name__ )
INITIAL_CHANGELOG_HASH = '000000000000'
def add_changeset( repo_ui, repo, path_to_filename_in_archive ):
commands.add( repo_ui, repo, path_to_filename_in_archive )
def bundle_to_json( fh ):
"""
Convert the received HG10xx data stream (a mercurial 1.0 bundle created using hg push from the
command line) to a json object.
"""
# See http://www.wstein.org/home/wstein/www/home/was/patches/hg_json
hg_unbundle10_obj = readbundle( fh, None )
groups = [ group for group in unpack_groups( hg_unbundle10_obj ) ]
return json.dumps( groups, indent=4 )
def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ):
"""
Clone the repository up to the specified changeset_revision. No subsequent revisions will be
@@ -39,6 +56,9 @@ def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ):
log.debug( error_message )
return False, error_message
def commit_changeset( repo_ui, repo, full_path_to_changeset, username, message ):
commands.commit( repo_ui, repo, full_path_to_changeset, user=username, message=message )
def copy_file_from_manifest( repo, ctx, filename, dir ):
"""
Copy the latest version of the file named filename from the repository manifest to the directory
@@ -268,6 +288,9 @@ def pull_repository( repo, repository_clone_url, ctx_rev ):
"""Pull changes from a remote repository to a local one."""
commands.pull( get_configured_ui(), repo, source=repository_clone_url, rev=[ ctx_rev ] )
def remove_file( repo_ui, repo, selected_file, force=True ):
commands.remove( repo_ui, repo, selected_file, force=force )
def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ):
"""
Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision,
@@ -300,6 +323,65 @@ def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revi
"""
return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision )
def unpack_chunks( hg_unbundle10_obj ):
"""
This method provides a generator of parsed chunks of a "group" in a mercurial unbundle10 object which
is created when a changeset that is pushed to a Tool Shed repository using hg push from the command line
is read using readbundle.
"""
while True:
length, = struct.unpack( '>l', readexactly( hg_unbundle10_obj, 4 ) )
if length <= 4:
# We found a "null chunk", which ends the group.
break
if length < 84:
raise Exception( "negative data length" )
node, p1, p2, cs = struct.unpack( '20s20s20s20s', readexactly( hg_unbundle10_obj, 80 ) )
yield { 'node': node.encode( 'hex' ),
'p1': p1.encode( 'hex' ),
'p2': p2.encode( 'hex' ),
'cs': cs.encode( 'hex' ),
'data': [ patch for patch in unpack_patches( hg_unbundle10_obj, length - 84 ) ] }
def unpack_groups( hg_unbundle10_obj ):
"""
This method provides a generator of parsed groups from a mercurial unbundle10 object which is
created when a changeset that is pushed to a Tool Shed repository using hg push from the command
line is read using readbundle.
"""
# Process the changelog group.
yield [ chunk for chunk in unpack_chunks( hg_unbundle10_obj ) ]
# Process the manifest group.
yield [ chunk for chunk in unpack_chunks( hg_unbundle10_obj ) ]
while True:
length, = struct.unpack( '>l', readexactly( hg_unbundle10_obj, 4 ) )
if length <= 4:
# We found a "null meta chunk", which ends the changegroup.
break
filename = readexactly( hg_unbundle10_obj, length-4 ).encode( 'string_escape' )
# Process the file group.
yield ( filename, [ chunk for chunk in unpack_chunks( hg_unbundle10_obj ) ] )
def unpack_patches( hg_unbundle10_obj, remaining ):
"""
This method provides a generator of patches from the data field in a chunk. As there is no delimiter
for this data field, a length argument is required.
"""
while remaining >= 12:
start, end, blocklen = struct.unpack( '>lll', readexactly( hg_unbundle10_obj, 12 ) )
remaining -= 12
if blocklen > remaining:
raise Exception( "unexpected end of patch stream" )
block = readexactly( hg_unbundle10_obj, blocklen )
remaining -= blocklen
yield { 'start': start,
'end': end,
'blocklen': blocklen,
'block': block.encode( 'string_escape' ) }
if remaining > 0:
print remaining
raise Exception( "unexpected end of patch stream" )
def update_repository( repo, ctx_rev=None ):
"""
Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired
lib/tool_shed/util/import_util.py
+101 -28
View File
@@ -5,50 +5,117 @@ import tarfile
import tempfile
import urllib
from galaxy import util
from galaxy.datatypes import checkers
from tool_shed.util import commit_util
from tool_shed.util import encoding_util
from tool_shed.util import hg_util
from tool_shed.util import metadata_util
from tool_shed.util import repository_maintenance_util
from tool_shed.util import xml_util
import tool_shed.util.shed_util_common as suc
import tool_shed.repository_types.util as rt_util
log = logging.getLogger( __name__ )
def check_status_and_reset_downloadable( trans, import_results_tups ):
def check_status_and_reset_downloadable( app, import_results_tups ):
"""Check the status of each imported repository and set downloadable to False if errors."""
sa_session = app.model.context.current
flush = False
for import_results_tup in import_results_tups:
ok, name_owner, message = import_results_tup
name, owner = name_owner
if not ok:
repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
repository = suc.get_repository_by_name_and_owner( app, name, owner )
if repository is not None:
# Do not allow the repository to be automatically installed if population resulted in errors.
tip_changeset_revision = repository.tip( trans.app )
repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app,
trans.security.encode_id( repository.id ),
tip_changeset_revision = repository.tip( app )
repository_metadata = suc.get_repository_metadata_by_changeset_revision( app,
app.security.encode_id( repository.id ),
tip_changeset_revision )
if repository_metadata:
if repository_metadata.downloadable:
repository_metadata.downloadable = False
trans.sa_session.add( repository_metadata )
sa_session.add( repository_metadata )
if not flush:
flush = True
# Do not allow dependent repository revisions to be automatically installed if population
# resulted in errors.
dependent_downloadable_revisions = suc.get_dependent_downloadable_revisions( trans.app, repository_metadata )
dependent_downloadable_revisions = suc.get_dependent_downloadable_revisions( app, repository_metadata )
for dependent_downloadable_revision in dependent_downloadable_revisions:
if dependent_downloadable_revision.downloadable:
dependent_downloadable_revision.downloadable = False
trans.sa_session.add( dependent_downloadable_revision )
sa_session.add( dependent_downloadable_revision )
if not flush:
flush = True
if flush:
trans.sa_session.flush()
sa_session.flush()
def extract_capsule_files( trans, **kwd ):
def create_repository_and_import_archive( app, host, user, repository_archive_dict, import_results_tups ):
"""
Create a new repository in the tool shed and populate it with the contents of a gzip compressed
tar archive that was exported as part or all of the contents of a capsule.
"""
results_message = ''
name = repository_archive_dict.get( 'name', None )
username = repository_archive_dict.get( 'owner', None )
if name is None or username is None:
ok = False
results_message += 'Import failed: required repository name <b>%s</b> or owner <b>%s</b> is missing.' % \
( str( name ), str( username ))
import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
else:
status = repository_archive_dict.get( 'status', None )
if status is None:
# The repository does not yet exist in this Tool Shed and the current user is authorized to import
# the current archive file.
type = repository_archive_dict.get( 'type', 'unrestricted' )
description = repository_archive_dict.get( 'description', '' )
long_description = repository_archive_dict.get( 'long_description', '' )
# The owner entry in the repository_archive_dict is the public username of the user associated with
# the exported repository archive.
user = suc.get_user_by_username( app, username )
if user is None:
ok = False
results_message += 'Import failed: repository owner <b>%s</b> does not have an account in this Tool Shed.' % str( username )
import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
else:
user_id = user.id
# The categories entry in the repository_archive_dict is a list of category names. If a name does not
# exist in the current Tool Shed, the category will not be created, so it will not be associated with
# the repository.
category_ids = []
category_names = repository_archive_dict.get( 'category_names', [] )
for category_name in category_names:
category = suc.get_category_by_name( app, category_name )
if category is None:
results_message += 'This Tool Shed does not have the category <b>%s</b> so it ' % str( category_name )
results_message += 'will not be associated with this repository.'
else:
category_ids.append( app.security.encode_id( category.id ) )
# Create the repository record in the database.
repository, create_message = repository_maintenance_util.create_repository( app,
name,
type,
description,
long_description,
user_id=user_id,
category_ids=category_ids )
if create_message:
results_message += create_message
# Populate the new repository with the contents of exported repository archive.
results_dict = import_repository_archive( app, host, user, repository, repository_archive_dict )
ok = results_dict.get( 'ok', False )
error_message = results_dict.get( 'error_message', '' )
if error_message:
results_message += error_message
import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
else:
# The repository either already exists in this Tool Shed or the current user is not authorized to create it.
ok = True
results_message += 'Import not necessary: repository status for this Tool Shed is: %s.' % str( status )
import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
return import_results_tups
def extract_capsule_files( **kwd ):
"""Extract the uploaded capsule archive into a temporary location for inspection, validation and potential import."""
return_dict = {}
tar_archive = kwd.get( 'tar_archive', None )
@@ -168,7 +235,7 @@ def get_repository_info_from_manifest( manifest_file_path ):
repository_info_dicts.append( repository_info_dict )
return repository_info_dicts, error_message
def get_repository_status_from_tool_shed( trans, repository_info_dicts ):
def get_repository_status_from_tool_shed( app, user, user_is_admin, repository_info_dicts ):
"""
For each exported repository archive contained in the capsule, inspect the Tool Shed to see if that repository already
exists or if the current user is authorized to create the repository, and set a status appropriately. If repository
@@ -179,7 +246,7 @@ def get_repository_status_from_tool_shed( trans, repository_info_dicts ):
"""
repository_status_info_dicts = []
for repository_info_dict in repository_info_dicts:
repository = suc.get_repository_by_name_and_owner( trans.app, repository_info_dict[ 'name' ], repository_info_dict[ 'owner' ] )
repository = suc.get_repository_by_name_and_owner( app, repository_info_dict[ 'name' ], repository_info_dict[ 'owner' ] )
if repository:
if repository.deleted:
repository_info_dict[ 'status' ] = 'Exists, deleted'
@@ -189,16 +256,16 @@ def get_repository_status_from_tool_shed( trans, repository_info_dicts ):
repository_info_dict[ 'status' ] = 'Exists'
else:
# No repository with the specified name and owner currently exists, so make sure the current user can create one.
if trans.user_is_admin():
if user_is_admin:
repository_info_dict[ 'status' ] = None
elif trans.app.security_agent.user_can_import_repository_archive( trans.user, repository_info_dict[ 'owner' ] ):
elif app.security_agent.user_can_import_repository_archive( user, repository_info_dict[ 'owner' ] ):
repository_info_dict[ 'status' ] = None
else:
repository_info_dict[ 'status' ] = 'Not authorized to import'
repository_status_info_dicts.append( repository_info_dict )
return repository_status_info_dicts
def import_repository_archive( trans, repository, repository_archive_dict ):
def import_repository_archive( app, host, user, repository, repository_archive_dict ):
"""Import a repository archive contained within a repository capsule."""
archive_file_name = repository_archive_dict.get( 'archive_file_name', None )
capsule_file_name = repository_archive_dict[ 'capsule_file_name' ]
@@ -207,8 +274,8 @@ def import_repository_archive( trans, repository, repository_archive_dict ):
results_dict = dict( ok=True, error_message='' )
archive_file_path = os.path.join( file_path, archive_file_name )
archive = tarfile.open( archive_file_path, 'r:*' )
repo_dir = repository.repo_path( trans.app )
repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False )
repo_dir = repository.repo_path( app )
repo = hg_util.get_repo_for_repository( app, repository=None, repo_path=repo_dir, create=False )
undesirable_dirs_removed = 0
undesirable_files_removed = 0
ok, error_message = commit_util.check_archive( repository, archive )
@@ -237,9 +304,8 @@ def import_repository_archive( trans, repository, repository_archive_dict ):
uploaded_file_name = os.path.join( full_path, filename )
if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans,
uploaded_file_name,
unpopulate=False )
altered, root_elem, error_message = \
commit_util.handle_repository_dependencies_definition( app, uploaded_file_name, unpopulate=False )
if error_message:
results_dict[ 'ok' ] = False
results_dict[ 'error_message' ] += error_message
@@ -248,7 +314,7 @@ def import_repository_archive( trans, repository, repository_archive_dict ):
shutil.move( tmp_filename, uploaded_file_name )
elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( app, uploaded_file_name )
if error_message:
results_dict[ 'ok' ] = False
results_dict[ 'error_message' ] += error_message
@@ -261,7 +327,9 @@ def import_repository_archive( trans, repository, repository_archive_dict ):
# Since the repository is new, the following must be False.
remove_repo_files_not_in_tar = False
ok, error_message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
commit_util.handle_directory_changes( trans,
commit_util.handle_directory_changes( app,
host,
user.username,
repository,
full_path,
filenames_in_archive,
@@ -274,9 +342,14 @@ def import_repository_archive( trans, repository, repository_archive_dict ):
results_dict[ 'ok' ] = False
results_dict[ 'error_message' ] += error_message
try:
metadata_util.set_repository_metadata_due_to_new_tip( trans,
repository,
content_alert_str=content_alert_str )
status, error_message = metadata_util.set_repository_metadata_due_to_new_tip( app,
host,
user,
repository,
content_alert_str=content_alert_str )
if error_message:
results_dict[ 'ok' ] = False
results_dict[ 'error_message' ] += error_message
except Exception, e:
log.debug( "Error setting metadata on repository %s created from imported archive %s: %s" % \
( str( repository.name ), str( archive_file_name ), str( e ) ) )
@@ -286,7 +359,7 @@ def import_repository_archive( trans, repository, repository_archive_dict ):
results_dict[ 'error_message' ] += error_message
return results_dict
def upload_capsule( trans, **kwd ):
def upload_capsule( **kwd ):
"""Upload and prepare an exported repository capsule for validation."""
file_data = kwd.get( 'file_data', '' )
url = kwd.get( 'url', '' )
@@ -349,7 +422,7 @@ def upload_capsule( trans, **kwd ):
return return_dict
return return_dict
def validate_capsule( trans, **kwd ):
def validate_capsule( **kwd ):
"""Inspect the uploaded capsule's manifest and its contained files to ensure it is a valid repository capsule."""
capsule_dict = {}
capsule_dict.update( kwd )
lib/tool_shed/util/metadata_util.py
+182 -149
View File
@@ -40,13 +40,14 @@ NOT_TOOL_CONFIGS = [ suc.DATATYPES_CONFIG_FILENAME,
rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME,
REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ]
def add_tool_versions( trans, id, repository_metadata, changeset_revisions ):
def add_tool_versions( app, id, repository_metadata, changeset_revisions ):
# Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata.
sa_session = app.model.context.current
metadata = repository_metadata.metadata
tool_versions_dict = {}
for tool_dict in metadata.get( 'tools', [] ):
# We have at least 2 changeset revisions to compare tool guids and tool ids.
parent_id = get_parent_id( trans.app,
parent_id = get_parent_id( app,
id,
tool_dict[ 'id' ],
tool_dict[ 'version' ],
@@ -55,8 +56,8 @@ def add_tool_versions( trans, id, repository_metadata, changeset_revisions ):
tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id
if tool_versions_dict:
repository_metadata.tool_versions = tool_versions_dict
trans.sa_session.add( repository_metadata )
trans.sa_session.flush()
sa_session.add( repository_metadata )
sa_session.flush()
def clean_repository_metadata( trans, id, changeset_revisions ):
# Delete all repository_metadata records associated with the repository that have a changeset_revision that is not in changeset_revisions.
@@ -72,10 +73,15 @@ def clean_repository_metadata( trans, id, changeset_revisions ):
trans.sa_session.delete( repository_metadata )
trans.sa_session.flush()
def compare_changeset_revisions( trans, ancestor_changeset_revision, ancestor_metadata_dict, current_changeset_revision, current_metadata_dict ):
"""Compare the contents of two changeset revisions to determine if a new repository metadata revision should be created."""
# The metadata associated with ancestor_changeset_revision is ancestor_metadata_dict. This changeset_revision is an ancestor of
# current_changeset_revision which is associated with current_metadata_dict. A new repository_metadata record will be created only
def compare_changeset_revisions( app, ancestor_changeset_revision, ancestor_metadata_dict,
current_changeset_revision, current_metadata_dict ):
"""
Compare the contents of two changeset revisions to determine if a new repository
metadata revision should be created.
"""
# The metadata associated with ancestor_changeset_revision is ancestor_metadata_dict.
# This changeset_revision is an ancestor of current_changeset_revision which is associated
# with current_metadata_dict. A new repository_metadata record will be created only
# when this method returns the constant value NOT_EQUAL_AND_NOT_SUBSET.
ancestor_datatypes = ancestor_metadata_dict.get( 'datatypes', [] )
ancestor_tools = ancestor_metadata_dict.get( 'tools', [] )
@@ -107,14 +113,16 @@ def compare_changeset_revisions( trans, ancestor_changeset_revision, ancestor_me
no_data_manager = not ancestor_data_manager and not current_data_manager
if no_datatypes and no_readme_files and no_repository_dependencies and no_tool_dependencies and no_tools and no_workflows and no_data_manager:
return NO_METADATA
# Uncomment the following if we decide that README files should affect how installable repository revisions are defined. See the NOTE in the
# compare_readme_files() method.
# Uncomment the following if we decide that README files should affect how installable
# repository revisions are defined. See the NOTE in the compare_readme_files() method.
# readme_file_comparision = compare_readme_files( ancestor_readme_files, current_readme_files )
repository_dependency_comparison = compare_repository_dependencies( trans, ancestor_repository_dependencies, current_repository_dependencies )
tool_dependency_comparison = compare_tool_dependencies( trans, ancestor_tool_dependencies, current_tool_dependencies )
workflow_comparison = compare_workflows( trans, ancestor_workflows, current_workflows )
datatype_comparison = compare_datatypes( trans, ancestor_datatypes, current_datatypes )
data_manager_comparison = compare_data_manager( trans, ancestor_data_manager, current_data_manager )
repository_dependency_comparison = compare_repository_dependencies( app,
ancestor_repository_dependencies,
current_repository_dependencies )
tool_dependency_comparison = compare_tool_dependencies( ancestor_tool_dependencies, current_tool_dependencies )
workflow_comparison = compare_workflows( ancestor_workflows, current_workflows )
datatype_comparison = compare_datatypes( ancestor_datatypes, current_datatypes )
data_manager_comparison = compare_data_manager( ancestor_data_manager, current_data_manager )
# Handle case where all metadata is the same.
if ancestor_guids == current_guids and \
repository_dependency_comparison == EQUAL and \
@@ -130,7 +138,11 @@ def compare_changeset_revisions( trans, ancestor_changeset_revision, ancestor_me
workflow_dependency_is_subset = workflow_comparison in SUBSET_VALUES
datatype_is_subset = datatype_comparison in SUBSET_VALUES
datamanager_is_subset = data_manager_comparison in SUBSET_VALUES
if repository_dependency_is_subset and tool_dependency_is_subset and workflow_dependency_is_subset and datatype_is_subset and datamanager_is_subset:
if repository_dependency_is_subset and \
tool_dependency_is_subset and \
workflow_dependency_is_subset and \
datatype_is_subset and \
datamanager_is_subset:
is_subset = True
for guid in ancestor_guids:
if guid not in current_guids:
@@ -140,11 +152,16 @@ def compare_changeset_revisions( trans, ancestor_changeset_revision, ancestor_me
return SUBSET
return NOT_EQUAL_AND_NOT_SUBSET
def compare_data_manager( trans, ancestor_metadata, current_metadata ):
def compare_data_manager( ancestor_metadata, current_metadata ):
"""Determine if ancestor_metadata is the same as or a subset of current_metadata for data_managers."""
def __data_manager_dict_to_tuple_list( metadata_dict ):
# we do not check tool_guid or tool conf file name
return set( sorted( [ ( name, tuple( sorted( value.get( 'data_tables', [] ) ) ), value.get( 'guid' ), value.get( 'version' ), value.get( 'name' ), value.get( 'id' ) ) for name, value in metadata_dict.iteritems() ] ) )
return set( sorted( [ ( name,
tuple( sorted( value.get( 'data_tables', [] ) ) ),
value.get( 'guid' ),
value.get( 'version' ),
value.get( 'name' ),
value.get( 'id' ) ) for name, value in metadata_dict.iteritems() ] ) )
# only compare valid entries, any invalid entries are ignored
ancestor_metadata = __data_manager_dict_to_tuple_list( ancestor_metadata.get( 'data_managers', {} ) )
current_metadata = __data_manager_dict_to_tuple_list( current_metadata.get( 'data_managers', {} ) )
@@ -155,7 +172,7 @@ def compare_data_manager( trans, ancestor_metadata, current_metadata ):
return SUBSET
return NOT_EQUAL_AND_NOT_SUBSET
def compare_datatypes( trans, ancestor_datatypes, current_datatypes ):
def compare_datatypes( ancestor_datatypes, current_datatypes ):
"""Determine if ancestor_datatypes is the same as or a subset of current_datatypes."""
# Each datatype dict looks something like: {"dtype": "galaxy.datatypes.images:Image", "extension": "pdf", "mimetype": "application/pdf"}
if len( ancestor_datatypes ) <= len( current_datatypes ):
@@ -199,8 +216,10 @@ def compare_readme_files( ancestor_readme_files, current_readme_files ):
return SUBSET
return NOT_EQUAL_AND_NOT_SUBSET
def compare_repository_dependencies( trans, ancestor_repository_dependencies, current_repository_dependencies ):
"""Determine if ancestor_repository_dependencies is the same as or a subset of current_repository_dependencies."""
def compare_repository_dependencies( app, ancestor_repository_dependencies, current_repository_dependencies ):
"""
Determine if ancestor_repository_dependencies is the same as or a subset of current_repository_dependencies.
"""
# The list of repository_dependencies looks something like:
# [["http://localhost:9009", "emboss_datatypes", "test", "ab03a2a5f407", "False", "False"]].
# Create a string from each tuple in the list for easier comparison.
@@ -223,9 +242,12 @@ def compare_repository_dependencies( trans, ancestor_repository_dependencies, cu
found_in_current = True
break
if not found_in_current:
# In some cases, the only difference between a dependency definition in the lists is the changeset_revision value. We'll
# check to see if this is the case, and if the defined dependency is a repository that has metadata set only on its tip.
if not different_revision_defines_tip_only_repository_dependency( trans, ancestor_tup, current_repository_dependencies ):
# In some cases, the only difference between a dependency definition in the lists
# is the changeset_revision value. We'll check to see if this is the case, and if
# the defined dependency is a repository that has metadata set only on its tip.
if not different_revision_defines_tip_only_repository_dependency( app,
ancestor_tup,
current_repository_dependencies ):
return NOT_EQUAL_AND_NOT_SUBSET
return SUBSET
if len( ancestor_repository_dependencies ) == len( current_repository_dependencies ):
@@ -234,16 +256,17 @@ def compare_repository_dependencies( trans, ancestor_repository_dependencies, cu
return SUBSET
return NOT_EQUAL_AND_NOT_SUBSET
def compare_tool_dependencies( trans, ancestor_tool_dependencies, current_tool_dependencies ):
def compare_tool_dependencies( ancestor_tool_dependencies, current_tool_dependencies ):
"""Determine if ancestor_tool_dependencies is the same as or a subset of current_tool_dependencies."""
# The tool_dependencies dictionary looks something like:
# {'bwa/0.5.9': {'readme': 'some string', 'version': '0.5.9', 'type': 'package', 'name': 'bwa'}}
if len( ancestor_tool_dependencies ) <= len( current_tool_dependencies ):
for ancestor_td_key, ancestor_requirements_dict in ancestor_tool_dependencies.items():
if ancestor_td_key in current_tool_dependencies:
# The only values that could have changed between the 2 dictionaries are the "readme" or "type" values. Changing the readme value
# makes no difference. Changing the type will change the installation process, but for now we'll assume it was a typo, so new metadata
# shouldn't be generated.
# The only values that could have changed between the 2 dictionaries are the
# "readme" or "type" values. Changing the readme value makes no difference.
# Changing the type will change the installation process, but for now we'll
# assume it was a typo, so new metadata shouldn't be generated.
continue
else:
return NOT_EQUAL_AND_NOT_SUBSET
@@ -254,8 +277,11 @@ def compare_tool_dependencies( trans, ancestor_tool_dependencies, current_tool_d
return SUBSET
return NOT_EQUAL_AND_NOT_SUBSET
def compare_workflows( trans, ancestor_workflows, current_workflows ):
"""Determine if ancestor_workflows is the same as current_workflows or if ancestor_workflows is a subset of current_workflows."""
def compare_workflows( ancestor_workflows, current_workflows ):
"""
Determine if ancestor_workflows is the same as current_workflows or if ancestor_workflows
is a subset of current_workflows.
"""
if len( ancestor_workflows ) <= len( current_workflows ):
for ancestor_workflow_tup in ancestor_workflows:
# ancestor_workflows is a list of tuples where each contained tuple is
@@ -267,8 +293,10 @@ def compare_workflows( trans, ancestor_workflows, current_workflows ):
found_in_current = False
for current_workflow_tup in current_workflows:
current_workflow_dict = current_workflow_tup[1]
# Assume that if the name and number of steps are euqal, then the workflows are the same. Of course, this may not be true...
if current_workflow_dict[ 'name' ] == ancestor_workflow_name and len( current_workflow_dict[ 'steps' ] ) == num_ancestor_workflow_steps:
# Assume that if the name and number of steps are euqal, then the workflows
# are the same. Of course, this may not be true...
if current_workflow_dict[ 'name' ] == ancestor_workflow_name and \
len( current_workflow_dict[ 'steps' ] ) == num_ancestor_workflow_steps:
found_in_current = True
break
if not found_in_current:
@@ -279,8 +307,9 @@ def compare_workflows( trans, ancestor_workflows, current_workflows ):
return SUBSET
return NOT_EQUAL_AND_NOT_SUBSET
def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ):
def create_or_update_repository_metadata( app, id, repository, changeset_revision, metadata_dict ):
"""Create or update a repository_metadatqa record in the tool shed."""
sa_session = app.model.context.current
has_repository_dependencies = False
has_repository_dependencies_only_if_compiling_contained_td = False
includes_datatypes = False
@@ -309,7 +338,7 @@ def create_or_update_repository_metadata( trans, id, repository, changeset_revis
downloadable = True
else:
downloadable = False
repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision )
repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, id, changeset_revision )
if repository_metadata:
# A repository metadata record already exists with the received changeset_revision, so we don't need to
# check the skip_tool_test table.
@@ -325,15 +354,15 @@ def create_or_update_repository_metadata( trans, id, repository, changeset_revis
# No repository_metadata record exists for the received changeset_revision, so we may need to update the
# skip_tool_test table.
check_skip_tool_test = True
repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id,
changeset_revision=changeset_revision,
metadata=metadata_dict,
downloadable=downloadable,
has_repository_dependencies=has_repository_dependencies,
includes_datatypes=includes_datatypes,
includes_tools=includes_tools,
includes_tool_dependencies=includes_tool_dependencies,
includes_workflows=includes_workflows )
repository_metadata = app.model.RepositoryMetadata( repository_id=repository.id,
changeset_revision=changeset_revision,
metadata=metadata_dict,
downloadable=downloadable,
has_repository_dependencies=has_repository_dependencies,
includes_datatypes=includes_datatypes,
includes_tools=includes_tools,
includes_tool_dependencies=includes_tool_dependencies,
includes_workflows=includes_workflows )
# Always set the default values for the following columns. When resetting all metadata on a repository
# this will reset the values.
repository_metadata.tools_functionally_correct = False
@@ -342,20 +371,20 @@ def create_or_update_repository_metadata( trans, id, repository, changeset_revis
repository_metadata.do_not_test = False
repository_metadata.time_last_tested = None
repository_metadata.tool_test_results = None
trans.sa_session.add( repository_metadata )
trans.sa_session.flush()
sa_session.add( repository_metadata )
sa_session.flush()
if check_skip_tool_test:
# Since we created a new repository_metadata record, we may need to update the skip_tool_test table to point to it. Inspect each
# changeset revision in the received repository's changelog (up to the received changeset revision) to see if it is contained in the
# skip_tool_test table. If it is, but is not associated with a repository_metadata record, reset that skip_tool_test record to the
# newly created repository_metadata record.
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False )
for changeset in repo.changelog:
changeset_hash = str( repo.changectx( changeset ) )
skip_tool_test = suc.get_skip_tool_test_by_changeset_revision( trans.app, changeset_hash )
skip_tool_test = suc.get_skip_tool_test_by_changeset_revision( app, changeset_hash )
if skip_tool_test:
# We found a skip_tool_test record associated with the changeset_revision, so see if it has a valid repository_revision.
repository_revision = get_repository_metadata_by_id( trans.app, trans.security.encode_id( repository_metadata.id ) )
repository_revision = get_repository_metadata_by_id( app, app.security.encode_id( repository_metadata.id ) )
if repository_revision:
# The skip_tool_test record is associated with a valid repository_metadata record, so proceed.
continue
@@ -364,14 +393,14 @@ def create_or_update_repository_metadata( trans, id, repository, changeset_revis
# require updating, so we won't break here, we'll continue to inspect the rest of the changelog up to the received
# changeset_revision.
skip_tool_test.repository_metadata_id = repository_metadata.id
trans.sa_session.add( skip_tool_test )
trans.sa_session.flush()
sa_session.add( skip_tool_test )
sa_session.flush()
if changeset_hash == changeset_revision:
# Proceed no further than the received changeset_revision.
break
return repository_metadata
def different_revision_defines_tip_only_repository_dependency( trans, rd_tup, repository_dependencies ):
def different_revision_defines_tip_only_repository_dependency( app, rd_tup, repository_dependencies ):
"""
Determine if the only difference between rd_tup and a dependency definition in the list of
repository_dependencies is the changeset_revision value.
@@ -386,8 +415,8 @@ def different_revision_defines_tip_only_repository_dependency( trans, rd_tup, re
cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed )
if cleaned_rd_tool_shed == cleaned_tool_shed and rd_name == name and rd_owner == owner:
# Determine if the repository represented by the dependency tuple is an instance of the repository type TipOnly.
required_repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
repository_type_class = trans.app.repository_types_registry.get_class_by_label( required_repository.type )
required_repository = suc.get_repository_by_name_and_owner( app, name, owner )
repository_type_class = app.repository_types_registry.get_class_by_label( required_repository.type )
return isinstance( repository_type_class, TipOnly )
return False
@@ -1045,20 +1074,17 @@ def generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_
metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ]
return metadata_dict
def get_latest_repository_metadata( trans, decoded_repository_id, downloadable=False ):
def get_latest_repository_metadata( app, decoded_repository_id, downloadable=False ):
"""Get last metadata defined for a specified repository from the database."""
repository = trans.sa_session.query( trans.model.Repository ).get( decoded_repository_id )
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
sa_session = app.model.context.current
repository = sa_session.query( app.model.Repository ).get( decoded_repository_id )
repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False )
if downloadable:
changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app,
repository,
repo )
changeset_revision = suc.get_latest_downloadable_changeset_revision( app, repository, repo )
else:
changeset_revision = suc.get_latest_changeset_revision( trans.app,
repository,
repo )
return suc.get_repository_metadata_by_changeset_revision( trans.app,
trans.security.encode_id( repository.id ),
changeset_revision = suc.get_latest_changeset_revision( app, repository, repo )
return suc.get_repository_metadata_by_changeset_revision( app,
app.security.encode_id( repository.id ),
changeset_revision )
def get_parent_id( app, id, old_id, version, guid, changeset_revisions ):
@@ -1377,10 +1403,11 @@ def is_malicious( app, id, changeset_revision, **kwd ):
return repository_metadata.malicious
return False
def new_datatypes_metadata_required( trans, repository_metadata, metadata_dict ):
def new_datatypes_metadata_required( repository_metadata, metadata_dict ):
"""
Compare the last saved metadata for each datatype in the repository with the new metadata in metadata_dict to determine if a new
repository_metadata table record is required or if the last saved metadata record can be updated for datatypes instead.
Compare the last saved metadata for each datatype in the repository with the new metadata
in metadata_dict to determine if a new repository_metadata table record is required or if
the last saved metadata record can be updated for datatypes instead.
"""
# Datatypes are stored in metadata as a list of dictionaries that looks like:
# [{'dtype': 'galaxy.datatypes.data:Text', 'subclass': 'True', 'extension': 'acedb'}]
@@ -1392,7 +1419,7 @@ def new_datatypes_metadata_required( trans, repository_metadata, metadata_dict )
if 'datatypes' in metadata:
ancestor_datatypes = metadata[ 'datatypes' ]
# The saved metadata must be a subset of the new metadata.
datatype_comparison = compare_datatypes( trans, ancestor_datatypes, current_datatypes )
datatype_comparison = compare_datatypes( ancestor_datatypes, current_datatypes )
if datatype_comparison == NOT_EQUAL_AND_NOT_SUBSET:
return True
else:
@@ -1409,23 +1436,28 @@ def new_datatypes_metadata_required( trans, repository_metadata, metadata_dict )
# The received metadata_dict includes no metadata for datatypes, so a new repository_metadata table record is not needed.
return False
def new_metadata_required_for_utilities( trans, repository, new_tip_metadata_dict ):
def new_metadata_required_for_utilities( app, repository, new_tip_metadata_dict ):
"""
Galaxy utilities currently consist of datatypes, repository_dependency definitions, tools, tool_dependency definitions and exported
Galaxy workflows. This method compares the last stored repository_metadata record associated with the received repository against the
contents of the received new_tip_metadata_dict and returns True or False for the union set of Galaxy utilities contained in both metadata
dictionaries. The metadata contained in new_tip_metadata_dict may not be a subset of that contained in the last stored repository_metadata
record associated with the received repository because one or more Galaxy utilities may have been deleted from the repository in the new tip.
Galaxy utilities currently consist of datatypes, repository_dependency definitions,
tools, tool_dependency definitions and exported Galaxy workflows. This method compares
the last stored repository_metadata record associated with the received repository against
the contents of the received new_tip_metadata_dict and returns True or False for the union
set of Galaxy utilities contained in both metadata dictionaries. The metadata contained
in new_tip_metadata_dict may not be a subset of that contained in the last stored
repository_metadata record associated with the received repository because one or more
Galaxy utilities may have been deleted from the repository in the new tip.
"""
repository_metadata = get_latest_repository_metadata( trans, repository.id, downloadable=False )
datatypes_required = new_datatypes_metadata_required( trans, repository_metadata, new_tip_metadata_dict )
# Uncomment the following if we decide that README files should affect how installable repository revisions are defined. See the NOTE in the
# compare_readme_files() method.
# readme_files_required = new_readme_files_metadata_required( trans, repository_metadata, new_tip_metadata_dict )
repository_dependencies_required = new_repository_dependency_metadata_required( trans, repository_metadata, new_tip_metadata_dict )
tools_required = new_tool_metadata_required( trans, repository_metadata, new_tip_metadata_dict )
tool_dependencies_required = new_tool_dependency_metadata_required( trans, repository_metadata, new_tip_metadata_dict )
workflows_required = new_workflow_metadata_required( trans, repository_metadata, new_tip_metadata_dict )
repository_metadata = get_latest_repository_metadata( app, repository.id, downloadable=False )
datatypes_required = new_datatypes_metadata_required( repository_metadata, new_tip_metadata_dict )
# Uncomment the following if we decide that README files should affect how installable
# repository revisions are defined. See the NOTE in the compare_readme_files() method.
# readme_files_required = new_readme_files_metadata_required( repository_metadata, new_tip_metadata_dict )
repository_dependencies_required = new_repository_dependency_metadata_required( app,
repository_metadata,
new_tip_metadata_dict )
tools_required = new_tool_metadata_required( repository_metadata, new_tip_metadata_dict )
tool_dependencies_required = new_tool_dependency_metadata_required( repository_metadata, new_tip_metadata_dict )
workflows_required = new_workflow_metadata_required( repository_metadata, new_tip_metadata_dict )
if datatypes_required or repository_dependencies_required or tools_required or tool_dependencies_required or workflows_required:
return True
return False
@@ -1462,7 +1494,7 @@ def new_readme_files_metadata_required( trans, repository_metadata, metadata_dic
# The received metadata_dict includes no metadata for readme_files, so a new repository_metadata table record is not needed.
return False
def new_repository_dependency_metadata_required( trans, repository_metadata, metadata_dict ):
def new_repository_dependency_metadata_required( app, repository_metadata, metadata_dict ):
"""
Compare the last saved metadata for each repository dependency in the repository with the new metadata in metadata_dict to determine if a new
repository_metadata table record is required or if the last saved metadata record can be updated for repository_dependencies instead.
@@ -1474,24 +1506,29 @@ def new_repository_dependency_metadata_required( trans, repository_metadata, met
new_repository_dependencies_metadata = metadata_dict.get( 'repository_dependencies', None )
if new_repository_dependencies_metadata:
new_repository_dependencies = metadata_dict[ 'repository_dependencies' ][ 'repository_dependencies' ]
# TODO: We used to include the following here to handle the case where repository dependency definitions were deleted. However,
# this erroneously returned True in cases where is should not have done so. This usually occurred where multiple single files
# were uploaded when a single tarball should have been. We need to implement support for handling deleted repository dependency
# definitions so that we can guarantee reproducibility, but we need to do it in a way that is better than the following.
# TODO: We used to include the following here to handle the case where repository
# dependency definitions were deleted. However this erroneously returned True in
# cases where is should not have done so. This usually occurred where multiple single
# files were uploaded when a single tarball should have been. We need to implement
# support for handling deleted repository dependency definitions so that we can guarantee
# reproducibility, but we need to do it in a way that is better than the following.
# for new_repository_dependency in new_repository_dependencies:
# if new_repository_dependency not in saved_repository_dependencies:
# return True
# The saved metadata must be a subset of the new metadata.
for saved_repository_dependency in saved_repository_dependencies:
if saved_repository_dependency not in new_repository_dependencies:
# In some cases, the only difference between a dependency definition in the lists is the changeset_revision value. We'll
# check to see if this is the case, and if the defined dependency is a repository that has metadata set only on its tip.
if not different_revision_defines_tip_only_repository_dependency( trans, saved_repository_dependency, new_repository_dependencies ):
# In some cases, the only difference between a dependency definition in the lists
# is the changeset_revision value. We'll check to see if this is the case, and if
# the defined dependency is a repository that has metadata set only on its tip.
if not different_revision_defines_tip_only_repository_dependency( app,
saved_repository_dependency,
new_repository_dependencies ):
return True
return False
else:
# The repository_dependencies.xml file must have been deleted, so create a new repository_metadata record so we always have
# access to the deleted file.
# The repository_dependencies.xml file must have been deleted, so create a new
# repository_metadata record so we always have access to the deleted file.
return True
else:
return False
@@ -1500,13 +1537,15 @@ def new_repository_dependency_metadata_required( trans, repository_metadata, met
# There is no saved repository metadata, so we need to create a new repository_metadata record.
return True
else:
# The received metadata_dict includes no metadata for repository dependencies, so a new repository_metadata record is not needed.
# The received metadata_dict includes no metadata for repository dependencies, so
# a new repository_metadata record is not needed.
return False
def new_tool_dependency_metadata_required( trans, repository_metadata, metadata_dict ):
def new_tool_dependency_metadata_required( repository_metadata, metadata_dict ):
"""
Compare the last saved metadata for each tool dependency in the repository with the new metadata in metadata_dict to determine if a new
repository_metadata table record is required or if the last saved metadata record can be updated for tool_dependencies instead.
Compare the last saved metadata for each tool dependency in the repository with the new
metadata in metadata_dict to determine if a new repository_metadata table record is required
or if the last saved metadata record can be updated for tool_dependencies instead.
"""
if repository_metadata:
metadata = repository_metadata.metadata
@@ -1545,10 +1584,11 @@ def new_tool_dependency_metadata_required( trans, repository_metadata, metadata_
# The received metadata_dict includes no metadata for tool dependencies, so a new repository_metadata record is not needed.
return False
def new_tool_metadata_required( trans, repository_metadata, metadata_dict ):
def new_tool_metadata_required( repository_metadata, metadata_dict ):
"""
Compare the last saved metadata for each tool in the repository with the new metadata in metadata_dict to determine if a new repository_metadata
table record is required, or if the last saved metadata record can be updated instead.
Compare the last saved metadata for each tool in the repository with the new metadata in
metadata_dict to determine if a new repository_metadata table record is required, or if
the last saved metadata record can be updated instead.
"""
if 'tools' in metadata_dict:
if repository_metadata:
@@ -1587,10 +1627,11 @@ def new_tool_metadata_required( trans, repository_metadata, metadata_dict ):
# The received metadata_dict includes no metadata for tools, so a new repository_metadata table record is not needed.
return False
def new_workflow_metadata_required( trans, repository_metadata, metadata_dict ):
def new_workflow_metadata_required( repository_metadata, metadata_dict ):
"""
Currently everything about an exported workflow except the name is hard-coded, so there's no real way to differentiate versions of
exported workflows. If this changes at some future time, this method should be enhanced accordingly.
Currently everything about an exported workflow except the name is hard-coded, so
there's no real way to differentiate versions of exported workflows. If this changes
at some future time, this method should be enhanced accordingly.
"""
if 'workflows' in metadata_dict:
if repository_metadata:
@@ -1819,7 +1860,7 @@ def reset_all_metadata_on_repository_in_tool_shed( trans, id ):
# SUBSET - ancestor metadata is a subset of current metadata, so continue from current
# NOT_EQUAL_AND_NOT_SUBSET - ancestor metadata is neither equal to nor a subset of current
# metadata, so persist ancestor metadata.
comparison = compare_changeset_revisions( trans,
comparison = compare_changeset_revisions( trans.app,
ancestor_changeset_revision,
ancestor_metadata_dict,
current_changeset_revision,
@@ -1830,7 +1871,7 @@ def reset_all_metadata_on_repository_in_tool_shed( trans, id ):
elif comparison == NOT_EQUAL_AND_NOT_SUBSET:
metadata_changeset_revision = ancestor_changeset_revision
metadata_dict = ancestor_metadata_dict
repository_metadata = create_or_update_repository_metadata( trans,
repository_metadata = create_or_update_repository_metadata( trans.app,
id,
repository,
metadata_changeset_revision,
@@ -1846,7 +1887,7 @@ def reset_all_metadata_on_repository_in_tool_shed( trans, id ):
metadata_changeset_revision = current_changeset_revision
metadata_dict = current_metadata_dict
# We're at the end of the change log.
repository_metadata = create_or_update_repository_metadata( trans,
repository_metadata = create_or_update_repository_metadata( trans.app,
id,
repository,
metadata_changeset_revision,
@@ -1858,7 +1899,7 @@ def reset_all_metadata_on_repository_in_tool_shed( trans, id ):
# We reach here only if current_metadata_dict is empty and ancestor_metadata_dict is not.
if not ctx.children():
# We're at the end of the change log.
repository_metadata = create_or_update_repository_metadata( trans,
repository_metadata = create_or_update_repository_metadata( trans.app,
id,
repository,
metadata_changeset_revision,
@@ -1901,7 +1942,7 @@ def reset_metadata_on_selected_repositories( trans, **kwd ):
owner = str( repository.owner )
invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans.app, repository_id )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False )
message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, repository, None, as_html=False )
log.debug( message )
unsuccessful_count += 1
else:
@@ -1947,20 +1988,21 @@ def set_add_to_tool_panel_attribute_for_tool( tool, guid, datatypes ):
return False
return True
def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
def set_repository_metadata( app, host, user, repository, content_alert_str='', **kwd ):
"""
Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the
repository owner that the changeset has problems.
Set metadata using the repository's current disk files, returning specific error
messages (if any) to alert the repository owner that the changeset has problems.
"""
sa_session = app.model.context.current
message = ''
status = 'done'
encoded_id = trans.security.encode_id( repository.id )
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository )
repo_dir = repository.repo_path( trans.app )
repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False )
metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app,
encoded_id = app.security.encode_id( repository.id )
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository )
repo_dir = repository.repo_path( app )
repo = hg_util.get_repo_for_repository( app, repository=None, repo_path=repo_dir, create=False )
metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=app,
repository=repository,
changeset_revision=repository.tip( trans.app ),
changeset_revision=repository.tip( app ),
repository_clone_url=repository_clone_url,
relative_install_dir=repo_dir,
repository_files_dir=None,
@@ -1969,25 +2011,25 @@ def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
persist=False )
if metadata_dict:
repository_metadata = None
repository_type_class = trans.app.repository_types_registry.get_class_by_label( repository.type )
repository_type_class = app.repository_types_registry.get_class_by_label( repository.type )
tip_only = isinstance( repository_type_class, TipOnly )
if not tip_only and new_metadata_required_for_utilities( trans, repository, metadata_dict ):
if not tip_only and new_metadata_required_for_utilities( app, repository, metadata_dict ):
# Create a new repository_metadata table row.
repository_metadata = create_or_update_repository_metadata( trans,
repository_metadata = create_or_update_repository_metadata( app,
encoded_id,
repository,
repository.tip( trans.app ),
repository.tip( app ),
metadata_dict )
# If this is the first record stored for this repository, see if we need to send any email alerts.
if len( repository.downloadable_revisions ) == 1:
suc.handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=True, admin_only=False )
suc.handle_email_alerts( app, host, repository, content_alert_str='', new_repo_alert=True, admin_only=False )
else:
# Update the latest stored repository metadata with the contents and attributes of metadata_dict.
repository_metadata = get_latest_repository_metadata( trans, repository.id, downloadable=False )
repository_metadata = get_latest_repository_metadata( app, repository.id, downloadable=False )
if repository_metadata:
downloadable = is_downloadable( metadata_dict )
# Update the last saved repository_metadata table row.
repository_metadata.changeset_revision = repository.tip( trans.app )
repository_metadata.changeset_revision = repository.tip( app )
repository_metadata.metadata = metadata_dict
repository_metadata.downloadable = downloadable
if 'datatypes' in metadata_dict:
@@ -2018,14 +2060,14 @@ def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
repository_metadata.tools_functionally_correct = False
repository_metadata.missing_test_components = False
repository_metadata.tool_test_results = None
trans.sa_session.add( repository_metadata )
trans.sa_session.flush()
sa_session.add( repository_metadata )
sa_session.flush()
else:
# There are no metadata records associated with the repository.
repository_metadata = create_or_update_repository_metadata( trans,
repository_metadata = create_or_update_repository_metadata( app,
encoded_id,
repository,
repository.tip( trans.app ),
repository.tip( app ),
metadata_dict )
if 'tools' in metadata_dict and repository_metadata and status != 'error':
# Set tool versions on the new downloadable change set. The order of the list of changesets is
@@ -2033,41 +2075,32 @@ def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
changeset_revisions = []
for changeset in repo.changelog:
changeset_revision = str( repo.changectx( changeset ) )
if suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_id, changeset_revision ):
if suc.get_repository_metadata_by_changeset_revision( app, encoded_id, changeset_revision ):
changeset_revisions.append( changeset_revision )
add_tool_versions( trans, encoded_id, repository_metadata, changeset_revisions )
add_tool_versions( app, encoded_id, repository_metadata, changeset_revisions )
elif len( repo ) == 1 and not invalid_file_tups:
message = "Revision <b>%s</b> includes no Galaxy utilities for which metadata can " % str( repository.tip( trans.app ) )
message = "Revision <b>%s</b> includes no Galaxy utilities for which metadata can " % str( repository.tip( app ) )
message += "be defined so this revision cannot be automatically installed into a local Galaxy instance."
status = "error"
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict )
message = tool_util.generate_message_for_invalid_tools( app, invalid_file_tups, repository, metadata_dict )
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables( trans.app )
tool_util.reset_tool_data_tables( app )
return message, status
def set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=None, **kwd ):
def set_repository_metadata_due_to_new_tip( app, host, user, repository, content_alert_str=None, **kwd ):
"""Set metadata on the repository tip in the tool shed."""
# This method is not called from Galaxy.
error_message, status = set_repository_metadata( trans,
repository,
content_alert_str=content_alert_str,
**kwd )
if error_message:
# FIXME: This probably should not redirect since this method is called from the upload controller as well
# as the repository controller.
# If there is an error, display it.
return trans.response.send_redirect( web.url_for( controller='repository',
action='manage_repository',
id=trans.security.encode_id( repository.id ),
message=error_message,
status='error' ) )
error_message, status = set_repository_metadata( app, host, user, repository, content_alert_str=content_alert_str, **kwd )
return status, error_message
def update_existing_tool_dependency( app, repository, original_dependency_dict, new_dependencies_dict ):
"""
Update an exsiting tool dependency whose definition was updated in a change set pulled by a Galaxy administrator when getting updates
to an installed tool shed repository. The original_dependency_dict is a single tool dependency definition, an example of which is::
Update an exsiting tool dependency whose definition was updated in a change set
pulled by a Galaxy administrator when getting updates to an installed tool shed
repository. The original_dependency_dict is a single tool dependency definition,
an example of which is::
{"name": "bwa",
"readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ",
lib/tool_shed/util/repository_maintenance_util.py
+46 -109
View File
@@ -5,7 +5,6 @@ import re
import tool_shed.util.shed_util_common as suc
from tool_shed.util import common_util
from tool_shed.util import hg_util
from tool_shed.util import import_util
from tool_shed.util import repository_dependency_util
from galaxy import util
from galaxy import web
@@ -38,12 +37,12 @@ def change_repository_name_in_hgrc_file( hgrc_file, new_name ):
config.write( new_file )
new_file.close()
def create_hgrc_file( trans, repository ):
def create_hgrc_file( app, repository ):
# At this point, an entry for the repository is required to be in the hgweb.config file so we can call repository.repo_path( trans.app ).
# Since we support both http and https, we set push_ssl to False to override the default (which is True) in the mercurial api. The hg
# purge extension purges all files and directories not being tracked by mercurial in the current repository. It'll remove unknown files
# and empty directories. This is not currently used because it is not supported in the mercurial API.
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False )
fp = repo.opener( 'hgrc', 'wb' )
fp.write( '[paths]\n' )
fp.write( 'default = .\n' )
@@ -129,20 +128,21 @@ def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_re
tool_dependencies )
return repo_info_dict
def create_repository( trans, name, type, description, long_description, user_id, category_ids=[] ):
def create_repository( app, name, type, description, long_description, user_id, category_ids=[] ):
sa_session = app.model.context.current
# Add the repository record to the database.
repository = trans.app.model.Repository( name=name,
type=type,
description=description,
long_description=long_description,
user_id=user_id )
repository = app.model.Repository( name=name,
type=type,
description=description,
long_description=long_description,
user_id=user_id )
# Flush to get the id.
trans.sa_session.add( repository )
trans.sa_session.flush()
sa_session.add( repository )
sa_session.flush()
# Create an admin role for the repository.
repository_admin_role = create_repository_admin_role( trans, repository )
repository_admin_role = create_repository_admin_role( app, repository )
# Determine the repository's repo_path on disk.
dir = os.path.join( trans.app.config.file_path, *directory_hash_id( repository.id ) )
dir = os.path.join( app.config.file_path, *directory_hash_id( repository.id ) )
# Create directory if it does not exist.
if not os.path.exists( dir ):
os.makedirs( dir )
@@ -152,112 +152,47 @@ def create_repository( trans, name, type, description, long_description, user_id
if not os.path.exists( repository_path ):
os.makedirs( repository_path )
# Create the local repository.
repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repository_path, create=True )
repo = hg_util.get_repo_for_repository( app, repository=None, repo_path=repository_path, create=True )
# Add an entry in the hgweb.config file for the local repository.
lhs = "repos/%s/%s" % ( repository.user.username, repository.name )
trans.app.hgweb_config_manager.add_entry( lhs, repository_path )
app.hgweb_config_manager.add_entry( lhs, repository_path )
# Create a .hg/hgrc file for the local repository.
create_hgrc_file( trans, repository )
create_hgrc_file( app, repository )
flush_needed = False
if category_ids:
# Create category associations
for category_id in category_ids:
category = trans.sa_session.query( trans.model.Category ) \
.get( trans.security.decode_id( category_id ) )
rca = trans.app.model.RepositoryCategoryAssociation( repository, category )
trans.sa_session.add( rca )
category = sa_session.query( app.model.Category ) \
.get( app.security.decode_id( category_id ) )
rca = app.model.RepositoryCategoryAssociation( repository, category )
sa_session.add( rca )
flush_needed = True
if flush_needed:
trans.sa_session.flush()
sa_session.flush()
# Update the repository registry.
trans.app.repository_registry.add_entry( repository )
app.repository_registry.add_entry( repository )
message = "Repository <b>%s</b> has been created." % str( repository.name )
return repository, message
def create_repository_admin_role( trans, repository ):
def create_repository_admin_role( app, repository ):
"""
Create a new role with name-spaced name based on the repository name and its owner's public user
name. This will ensure that the tole name is unique.
"""
sa_session = app.model.context.current
name = get_repository_admin_role_name( str( repository.name ), str( repository.user.username ) )
description = 'A user or group member with this role can administer this repository.'
role = trans.app.model.Role( name=name, description=description, type=trans.app.model.Role.types.SYSTEM )
trans.sa_session.add( role )
trans.sa_session.flush()
role = app.model.Role( name=name, description=description, type=app.model.Role.types.SYSTEM )
sa_session.add( role )
sa_session.flush()
# Associate the role with the repository owner.
ura = trans.model.UserRoleAssociation( repository.user, role )
ura = app.model.UserRoleAssociation( repository.user, role )
# Associate the role with the repository.
rra = trans.model.RepositoryRoleAssociation( repository, role )
trans.sa_session.add( rra )
trans.sa_session.flush()
rra = app.model.RepositoryRoleAssociation( repository, role )
sa_session.add( rra )
sa_session.flush()
return role
def create_repository_and_import_archive( trans, repository_archive_dict, import_results_tups ):
"""
Create a new repository in the tool shed and populate it with the contents of a gzip compressed
tar archive that was exported as part or all of the contents of a capsule.
"""
results_message = ''
name = repository_archive_dict.get( 'name', None )
username = repository_archive_dict.get( 'owner', None )
if name is None or username is None:
ok = False
results_message += 'Import failed: required repository name <b>%s</b> or owner <b>%s</b> is missing.' % \
( str( name ), str( username ))
import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
else:
status = repository_archive_dict.get( 'status', None )
if status is None:
# The repository does not yet exist in this Tool Shed and the current user is authorized to import
# the current archive file.
type = repository_archive_dict.get( 'type', 'unrestricted' )
description = repository_archive_dict.get( 'description', '' )
long_description = repository_archive_dict.get( 'long_description', '' )
# The owner entry in the repository_archive_dict is the public username of the user associated with
# the exported repository archive.
user = suc.get_user_by_username( trans.app, username )
if user is None:
ok = False
results_message += 'Import failed: repository owner <b>%s</b> does not have an account in this Tool Shed.' % str( username )
import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
else:
user_id = user.id
# The categories entry in the repository_archive_dict is a list of category names. If a name does not
# exist in the current Tool Shed, the category will not be created, so it will not be associated with
# the repository.
category_ids = []
category_names = repository_archive_dict.get( 'category_names', [] )
for category_name in category_names:
category = suc.get_category_by_name( trans.app, category_name )
if category is None:
results_message += 'This Tool Shed does not have the category <b>%s</b> so it ' % str( category_name )
results_message += 'will not be associated with this repository.'
else:
category_ids.append( trans.security.encode_id( category.id ) )
# Create the repository record in the database.
repository, create_message = create_repository( trans,
name,
type,
description,
long_description,
user_id=user_id,
category_ids=category_ids )
if create_message:
results_message += create_message
# Populate the new repository with the contents of exported repository archive.
results_dict = import_util.import_repository_archive( trans, repository, repository_archive_dict )
ok = results_dict.get( 'ok', False )
error_message = results_dict.get( 'error_message', '' )
if error_message:
results_message += error_message
import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
else:
# The repository either already exists in this Tool Shed or the current user is not authorized to create it.
ok = True
results_message += 'Import not necessary: repository status for this Tool Shed is: %s.' % str( status )
import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
return import_results_tups
def get_repo_info_dict( app, user, repository_id, changeset_revision ):
repository = suc.get_repository_in_tool_shed( app, repository_id )
repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False )
@@ -317,17 +252,19 @@ def get_repo_info_dict( app, user, repository_id, changeset_revision ):
def get_repository_admin_role_name( repository_name, repository_owner ):
return '%s_%s_admin' % ( str( repository_name ), str( repository_owner ) )
def get_role_by_id( trans, role_id ):
def get_role_by_id( app, role_id ):
"""Get a Role from the database by id."""
return trans.sa_session.query( trans.model.Role ).get( trans.security.decode_id( role_id ) )
sa_session = app.model.context.current
return sa_session.query( app.model.Role ).get( app.security.decode_id( role_id ) )
def handle_role_associations( trans, role, repository, **kwd ):
def handle_role_associations( app, role, repository, **kwd ):
sa_session = app.model.context.current
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository_owner = repository.user
if kwd.get( 'manage_role_associations_button', False ):
in_users_list = util.listify( kwd.get( 'in_users', [] ) )
in_users = [ trans.sa_session.query( trans.app.model.User ).get( x ) for x in in_users_list ]
in_users = [ sa_session.query( app.model.User ).get( x ) for x in in_users_list ]
# Make sure the repository owner is always associated with the repostory's admin role.
owner_associated = False
for user in in_users:
@@ -339,29 +276,29 @@ def handle_role_associations( trans, role, repository, **kwd ):
message += "The repository owner must always be associated with the repository's administrator role. "
status = 'error'
in_groups_list = util.listify( kwd.get( 'in_groups', [] ) )
in_groups = [ trans.sa_session.query( trans.app.model.Group ).get( x ) for x in in_groups_list ]
in_groups = [ sa_session.query( app.model.Group ).get( x ) for x in in_groups_list ]
in_repositories = [ repository ]
trans.app.security_agent.set_entity_role_associations( roles=[ role ],
app.security_agent.set_entity_role_associations( roles=[ role ],
users=in_users,
groups=in_groups,
repositories=in_repositories )
trans.sa_session.refresh( role )
sa_session.refresh( role )
message += "Role <b>%s</b> has been associated with %d users, %d groups and %d repositories. " % \
( str( role.name ), len( in_users ), len( in_groups ), len( in_repositories ) )
in_users = []
out_users = []
in_groups = []
out_groups = []
for user in trans.sa_session.query( trans.app.model.User ) \
.filter( trans.app.model.User.table.c.deleted==False ) \
.order_by( trans.app.model.User.table.c.email ):
for user in sa_session.query( app.model.User ) \
.filter( app.model.User.table.c.deleted==False ) \
.order_by( app.model.User.table.c.email ):
if user in [ x.user for x in role.users ]:
in_users.append( ( user.id, user.email ) )
else:
out_users.append( ( user.id, user.email ) )
for group in trans.sa_session.query( trans.app.model.Group ) \
.filter( trans.app.model.Group.table.c.deleted==False ) \
.order_by( trans.app.model.Group.table.c.name ):
for group in sa_session.query( app.model.Group ) \
.filter( app.model.Group.table.c.deleted==False ) \
.order_by( app.model.Group.table.c.name ):
if group in [ x.group for x in role.groups ]:
in_groups.append( ( group.id, group.name ) )
else:
lib/tool_shed/util/shed_util_common.py
+16 -15
View File
@@ -1231,7 +1231,7 @@ def get_user_by_username( app, username ):
except Exception, e:
return None
def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=False, admin_only=False ):
def handle_email_alerts( app, host, repository, content_alert_str='', new_repo_alert=False, admin_only=False ):
"""
There are 2 complementary features that enable a tool shed user to receive email notification:
1. Within User Preferences, they can elect to receive email when the first (or first valid)
@@ -1254,17 +1254,18 @@ def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert
user is not an admin user, the email will not include any information about both HTML and image content
that was included in the change set.
"""
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
sa_session = app.model.context.current
repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False )
sharable_link = generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None )
smtp_server = trans.app.config.smtp_server
smtp_server = app.config.smtp_server
if smtp_server and ( new_repo_alert or repository.email_alerts ):
# Send email alert to users that want them.
if trans.app.config.email_from is not None:
email_from = trans.app.config.email_from
elif trans.request.host.split( ':' )[0] == 'localhost':
if app.config.email_from is not None:
email_from = app.config.email_from
elif host.split( ':' )[0] == 'localhost':
email_from = 'galaxy-no-reply@' + socket.getfqdn()
else:
email_from = 'galaxy-no-reply@' + trans.request.host.split( ':' )[0]
email_from = 'galaxy-no-reply@' + host.split( ':' )[0]
tip_changeset = repo.changelog.tip()
ctx = repo.changectx( tip_changeset )
try:
@@ -1278,7 +1279,7 @@ def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert
else:
template = email_alert_template
display_date = hg_util.get_readable_ctx_date( ctx )
admin_body = string.Template( template ).safe_substitute( host=trans.request.host,
admin_body = string.Template( template ).safe_substitute( host=host,
sharable_link=sharable_link,
repository_name=repository.name,
revision='%s:%s' %( str( ctx.rev() ), ctx ),
@@ -1286,7 +1287,7 @@ def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert
description=ctx.description(),
username=username,
content_alert_str=content_alert_str )
body = string.Template( template ).safe_substitute( host=trans.request.host,
body = string.Template( template ).safe_substitute( host=host,
sharable_link=sharable_link,
repository_name=repository.name,
revision='%s:%s' %( str( ctx.rev() ), ctx ),
@@ -1294,15 +1295,15 @@ def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert
description=ctx.description(),
username=username,
content_alert_str='' )
admin_users = trans.app.config.get( "admin_users", "" ).split( "," )
admin_users = app.config.get( "admin_users", "" ).split( "," )
frm = email_from
if new_repo_alert:
subject = "Galaxy tool shed alert for new repository named %s" % str( repository.name )
subject = subject[ :80 ]
email_alerts = []
for user in trans.sa_session.query( trans.model.User ) \
.filter( and_( trans.model.User.table.c.deleted == False,
trans.model.User.table.c.new_repo_alert == True ) ):
for user in sa_session.query( app.model.User ) \
.filter( and_( app.model.User.table.c.deleted == False,
app.model.User.table.c.new_repo_alert == True ) ):
if admin_only:
if user.email in admin_users:
email_alerts.append( user.email )
@@ -1316,9 +1317,9 @@ def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert
# Send it
try:
if to in admin_users:
util.send_mail( frm, to, subject, admin_body, trans.app.config )
util.send_mail( frm, to, subject, admin_body, app.config )
else:
util.send_mail( frm, to, subject, body, trans.app.config )
util.send_mail( frm, to, subject, body, app.config )
except Exception, e:
log.exception( "An error occurred sending a tool shed repository update alert by email." )
lib/tool_shed/util/tool_util.py
+4 -4
View File
@@ -239,7 +239,7 @@ def copy_sample_files( app, sample_files, tool_path=None, sample_files_copied=No
if is_data_index_sample_file( filename ):
copy_sample_file( app, filename, dest_path=dest_path )
def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ):
def generate_message_for_invalid_tools( app, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ):
if as_html:
new_line = '<br/>'
bold_start = '<b>'
@@ -249,10 +249,10 @@ def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, me
bold_start = ''
bold_end = ''
message = ''
if trans.webapp.name == 'galaxy':
if app.name == 'galaxy':
tip_rev = str( repository.changeset_revision )
else:
tip_rev = str( repository.tip( trans.app ) )
tip_rev = str( repository.tip( app ) )
if not displaying_invalid_tool:
if metadata_dict:
message += "Metadata may have been defined for some items in revision '%s'. " % tip_rev
@@ -883,7 +883,7 @@ def load_tool_from_changeset_revision( trans, repository_id, changeset_revision,
tool,
sample_files )
if invalid_files_and_errors_tups:
message2 = generate_message_for_invalid_tools( trans,
message2 = generate_message_for_invalid_tools( trans.app,
invalid_files_and_errors_tups,
repository,
metadata_dict=None,