opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1918 from davebx/move_tsg

Browse Source 
Rename tool_shed_get to url_get.
This commit is contained in:
John Chilton
2016-03-25 10:57:03 -04:00
parent 6e34858e0d 7e14f5db62
commit ec3fe67ddd
19 changed files with 111 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/util/__init__.py
+41
View File
@@ -30,6 +30,7 @@ from xml.etree import ElementInclude, ElementTree
from six import binary_type, iteritems, PY3, string_types, text_type
from six.moves import email_mime_text, xrange, zip
from six.moves.urllib import parse as urlparse
from six.moves.urllib import request as urlrequest
try:
import docutils.core as docutils_core
@@ -1342,6 +1343,46 @@ def parse_int(value, min_val=None, max_val=None, default=None, allow_none=False)
raise
def build_url( base_url, port=80, scheme='http', pathspec=None, params=None, doseq=False ):
if params is None:
params = dict()
if pathspec is None:
pathspec = []
parsed_url = urlparse.urlparse( base_url )
if scheme != 'http':
parsed_url.scheme = scheme
if port != 80:
url = '%s://%s:%d/%s' % ( parsed_url.scheme, parsed_url.netloc.rstrip( '/' ), int( port ), parsed_url.path )
else:
url = '%s://%s/%s' % ( parsed_url.scheme, parsed_url.netloc.rstrip( '/' ), parsed_url.path.lstrip( '/' ) )
if len( pathspec ) > 0:
url = '%s/%s' % ( url.rstrip( '/' ), '/'.join( pathspec ) )
if parsed_url.query:
for query_parameter in parsed_url.query.split( '&' ):
key, value = query_parameter.split( '=' )
params[ key ] = value
if params:
url += '?%s' % urlparse.urlencode( params, doseq=doseq )
return url
def url_get( base_url, password_mgr=None, pathspec=None, params=None ):
"""Make contact with the uri provided and return any contents."""
# Uses system proxy settings if they exist.
proxy = urlrequest.ProxyHandler()
if password_mgr is not None:
auth = urlrequest.HTTPDigestAuthHandler( password_mgr )
urlopener = urlrequest.build_opener( proxy, auth )
else:
urlopener = urlrequest.build_opener( proxy )
urlrequest.install_opener( urlopener )
full_url = build_url( base_url, pathspec=pathspec, params=params )
response = urlopener.open( full_url )
content = response.read()
response.close()
return content
def safe_relpath(path):
"""
Given what we expect to be a relative path, determine whether the path
lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py
+1 -2
View File
@@ -13,7 +13,6 @@ from galaxy.web.base.controller import BaseAPIController
from tool_shed.galaxy_install.install_manager import InstallRepositoryManager
from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager
from tool_shed.util import common_util
from tool_shed.util import encoding_util
from tool_shed.util import hg_util
from tool_shed.util import workflow_util
@@ -100,7 +99,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
params = dict(name=name, owner=owner)
pathspec = ['api', 'repositories', 'get_ordered_installable_revisions']
try:
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec, params )
raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
except Exception, e:
message = "Error attempting to retrieve the latest installable revision from tool shed %s for repository %s owned by %s: %s" % \
( str( tool_shed_url ), str( name ), str( owner ), str( e ) )
lib/galaxy/webapps/galaxy/controllers/admin.py
+4 -4
View File
@@ -11,7 +11,7 @@ from galaxy.actions.admin import AdminActions
from galaxy.exceptions import MessageException
from galaxy.model import tool_shed_install as install_model
from galaxy.model.util import pgcalc
from galaxy.util import nice_size, sanitize_text
from galaxy.util import nice_size, sanitize_text, url_get
from galaxy.util.odict import odict
from galaxy.web import url_for
from galaxy.web.base.controller import BaseUIController, UsesQuotaMixin
@@ -770,9 +770,9 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
tree = galaxy.util.parse_xml( tools_xml_file_path )
root = tree.getroot()
tool_shed = root.get( 'name' )
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed )
shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed )
repo_name_dependency_tups = []
if tool_shed_url:
if shed_url:
for elem in root:
if elem.tag == 'repository':
tool_dependencies = []
@@ -781,7 +781,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
changeset_revision = elem.get( 'changeset_revision' )
params = dict( name=repository_name, owner='devteam', changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_tool_dependencies' ]
text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
text = url_get( shed_url, password_mgr=self.app.tool_shed_registry.url_auth( shed_url ), pathspec=pathspec, params=params )
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode( text )
for dependency_key, requirements_dict in tool_dependencies_dict.items():
lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+15 -15
View File
@@ -189,7 +189,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = kwd.get( 'tool_shed_url', '' )
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params )
url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -212,7 +212,7 @@ class AdminToolshed( AdminGalaxy ):
owner=str( repository.owner ),
changeset_revision=str( repository.changeset_revision ) )
pathspec = [ 'repository', 'check_for_updates' ]
url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -369,7 +369,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = kwd.get( 'tool_shed_url', '' )
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'find_tools' ], params=params )
url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'find_tools' ], params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -378,7 +378,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = kwd.get( 'tool_shed_url', '' )
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'find_workflows' ], params=params )
url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'find_workflows' ], params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -413,7 +413,7 @@ class AdminToolshed( AdminGalaxy ):
raise Exception( message )
params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_tool_dependencies' ]
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if len( raw_text ) > 2:
encoded_text = json.loads( raw_text )
text = encoding_util.tool_shed_decode( encoded_text )
@@ -439,7 +439,7 @@ class AdminToolshed( AdminGalaxy ):
owner=str( repository_owner ),
changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_updated_repository_information' ]
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
repo_information_dict = json.loads( raw_text )
return repo_information_dict
@@ -529,8 +529,8 @@ class AdminToolshed( AdminGalaxy ):
name=name,
owner=owner )
pathspec = [ 'repository', 'get_latest_downloadable_changeset_revision' ]
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
latest_downloadable_revision = json.loads( raw_text )
if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH:
message = 'Error retrieving the latest downloadable revision for this repository via the url <b>%s</b>.' % url
@@ -541,7 +541,7 @@ class AdminToolshed( AdminGalaxy ):
# appropriate repository revision if one exists. We need to create a temporary repo_info_tuple
# with the following entries to handle this.
# ( description, clone_url, changeset_revision, ctx_rev, owner, repository_dependencies, tool_dependencies )
tmp_clone_url = common_util.url_join( tool_shed_url, pathspec=[ 'repos', owner, name ] )
tmp_clone_url = util.build_url( tool_shed_url, pathspec=[ 'repos', owner, name ] )
tmp_repo_info_tuple = ( None, tmp_clone_url, latest_downloadable_revision, None, owner, None, None )
installed_repository, installed_changeset_revision = \
suc.repository_was_previously_installed( trans.app, tool_shed_url, name, tmp_repo_info_tuple, from_tip=False )
@@ -559,7 +559,7 @@ class AdminToolshed( AdminGalaxy ):
changeset_revisions=str( latest_downloadable_revision ),
galaxy_url=web.url_for( '/', qualified=True ) )
pathspec = [ 'repository', 'install_repositories_by_revision' ]
url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
else:
message = 'Cannot locate installed tool shed repository with encoded id <b>%s</b>.' % str( repository_id )
@@ -727,7 +727,7 @@ class AdminToolshed( AdminGalaxy ):
changeset_revisions=installed_changeset_revision,
galaxy_url=web.url_for( '/', qualified=True ) )
pathspec = [ 'repository', 'install_repositories_by_revision' ]
url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
description = kwd.get( 'description', repository.description )
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository )
@@ -985,7 +985,7 @@ class AdminToolshed( AdminGalaxy ):
try:
params = dict( name=str( repository.name ), owner=str( repository.owner ) )
pathspec = [ 'repository', 'get_repository_id' ]
repository_ids = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
repository_ids = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
except Exception, e:
# The Tool Shed cannot handle the get_repository_id request, so the code must be older than the
# 04/2014 Galaxy release when it was introduced. It will be safest to error out and let the
@@ -1005,7 +1005,7 @@ class AdminToolshed( AdminGalaxy ):
# Get the information necessary to install each repository.
params = dict( repository_ids=str( repository_ids ), changeset_revisions=str( changeset_revisions ) )
pathspec = [ 'repository', 'get_repository_information' ]
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
repo_information_dict = json.loads( raw_text )
for encoded_repo_info_dict in repo_information_dict.get( 'repo_info_dicts', [] ):
decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict )
@@ -1539,7 +1539,7 @@ class AdminToolshed( AdminGalaxy ):
owner=tool_shed_repository.owner,
changeset_revision=tool_shed_repository.installed_changeset_revision )
pathspec = [ 'repository', 'get_readme_files' ]
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
readme_files_dict = json.loads( raw_text )
tool_dependencies = metadata.get( 'tool_dependencies', None )
rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app )
@@ -1739,7 +1739,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) )
params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision )
pathspec = [ 'repository', 'get_tool_versions' ]
text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
tool_version_dicts = json.loads( text )
tvm = tool_version_manager.ToolVersionManager( trans.app )
lib/galaxy/webapps/galaxy/controllers/workflow.py
+1 -2
View File
@@ -10,7 +10,6 @@ from sqlalchemy import and_
from sqlalchemy.sql import expression
from markupsafe import escape
from tool_shed.util import common_util
from tool_shed.util import encoding_util
from galaxy import model
@@ -862,7 +861,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
workflow_name=encoding_util.tool_shed_encode( workflow_name ),
open_for_url=True )
pathspec = [ 'workflow', 'import_workflow' ]
workflow_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
workflow_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
import_button = True
if import_button:
workflow_data = None
lib/galaxy/webapps/tool_shed/controllers/repository.py
+4 -4
View File
@@ -809,7 +809,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
return update
params['latest_changeset_revision'] = str( latest_changeset_revision )
params['latest_ctx_rev'] = str( update_to_ctx.rev() )
url = common_util.url_join( galaxy_url, pathspec=pathspec, params=params )
url = util.build_url( galaxy_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -1065,7 +1065,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
trans.sa_session.flush()
tool_shed_url = web.url_for( '/', qualified=True )
pathspec = [ 'repos', str( repository.user.username ), str( repository.name ), 'archive', file_type_str ]
download_url = common_util.url_join( tool_shed_url, pathspec=pathspec )
download_url = util.build_url( tool_shed_url, pathspec=pathspec )
return trans.response.send_redirect( download_url )
@web.expose
@@ -1534,7 +1534,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
time_tested = repository_metadata.time_last_tested.strftime( '%a, %d %b %Y %H:%M:%S UT' )
# Generate a citable URL for this repository with owner and changeset revision.
pathspec = [ 'view', str( user.username ), str( repository.name ), str( repository_metadata.changeset_revision ) ]
repository_citable_url = common_util.url_join( tool_shed_url, pathspec=pathspec )
repository_citable_url = util.build_url( tool_shed_url, pathspec=pathspec )
passed_tests = len( tool_test_results.get( 'passed_tests', [] ) )
failed_tests = len( tool_test_results.get( 'failed_tests', [] ) )
missing_test_components = len( tool_test_results.get( 'missing_test_components', [] ) )
@@ -1982,7 +1982,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
repository_ids=','.join( util.listify( repository_ids ) ),
changeset_revisions=','.join( util.listify( changeset_revisions ) ) )
pathspec = [ 'admin_toolshed', 'prepare_for_install' ]
url = common_util.url_join( galaxy_url, pathspec=pathspec, params=params )
url = util.build_url( galaxy_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
else:
message = 'Repository installation is not possible due to an invalid Galaxy URL: <b>%s</b>. ' % galaxy_url
lib/tool_shed/capsule/capsule_manager.py
+2 -1
View File
@@ -15,6 +15,7 @@ from galaxy import web
from galaxy.util import asbool
from galaxy.util import CHUNK_SIZE
from galaxy.util import safe_relpath
from galaxy.util import build_url
from galaxy.util.odict import odict
from tool_shed.dependencies.repository.relation_builder import RelationBuilder
from tool_shed.dependencies import attribute_handlers
@@ -123,7 +124,7 @@ class ExportRepositoryManager( object ):
params = dict( encoded_repositories_archive_name=encoded_repositories_archive_name )
pathspec = [ 'repository', 'export_via_api' ]
tool_shed_url = web.url_for( '/', qualified=True )
download_url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
download_url = build_url( tool_shed_url, pathspec=pathspec, params=params )
return dict( download_url=download_url, error_messages=error_messages )
return repositories_archive, error_messages
lib/tool_shed/galaxy_install/dependency_display.py
+1 -1
View File
@@ -455,7 +455,7 @@ class DependencyDisplayer( object ):
owner=str( repository.owner ),
changeset_revision=str( repository.installed_changeset_revision ) )
pathspec = [ 'repository', 'get_readme_files' ]
raw_text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
readme_files_dict = json.loads( raw_text )
else:
readme_files_dict = readme_util.build_readme_files_dict( self.app,
lib/tool_shed/galaxy_install/install_manager.py
+3 -3
View File
@@ -460,7 +460,7 @@ class InstallRepositoryManager( object ):
changeset_revision=str( changeset_revision ) )
pathspec = [ 'api', 'repositories', 'get_repository_revision_install_info' ]
try:
raw_text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
except Exception, e:
message = "Error attempting to retrieve installation information from tool shed "
message += "%s for revision %s of repository %s owned by %s: %s" % \
@@ -1005,8 +1005,8 @@ def fetch_tool_versions( app, tool_shed_repository ):
owner=str( tool_shed_repository.owner ),
changeset_revision=str( tool_shed_repository.changeset_revision ) )
pathspec = [ 'repository', 'get_tool_versions' ]
url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
return json.loads( text )
else:
lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py
+4 -4
View File
@@ -8,7 +8,7 @@ import os
import urllib
import urllib2
from galaxy.util import asbool
from galaxy.util import asbool, url_get, build_url
from tool_shed.galaxy_install.tools import tool_panel_manager
@@ -291,9 +291,9 @@ class RepositoryDependencyInstallManager( object ):
changeset_revision=str( repository.changeset_revision ) )
pathspec = [ 'repository', 'get_repository_dependencies' ]
try:
raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
except Exception, e:
log.error("The URL\n%s\nraised the exception:\n%s\n", common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ), str( e ) )
log.error("The URL\n%s\nraised the exception:\n%s\n", build_url( tool_shed_url, pathspec=pathspec, params=params ), str( e ) )
return ''
if len( raw_text ) > 2:
encoded_text = json.loads( raw_text )
@@ -378,7 +378,7 @@ class RepositoryDependencyInstallManager( object ):
# Handle secure / insecure Tool Shed URL protocol changes and port changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
pathspec = [ 'repository', 'get_required_repo_info_dict' ]
url = common_util.url_join( tool_shed_url, pathspec=pathspec )
url = build_url( tool_shed_url, pathspec=pathspec )
# Fix for handling 307 redirect not being handled nicely by urllib2.urlopen when the urllib2.Request has data provided
url = urllib2.urlopen( urllib2.Request( url ) ).geturl()
request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
+2 -2
View File
@@ -3,7 +3,7 @@ import os
import tempfile
from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY
from galaxy.util import listify
from galaxy.util import listify, url_get
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import shed_util_common as suc
@@ -249,7 +249,7 @@ class Repository( RecipeTag, SyncDatabase ):
owner=owner,
changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_tool_dependencies_config_contents' ]
text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params )
text = url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
# Write the contents to a temporary file on disk so it can be reloaded and parsed.
fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-cttdc" )
lib/tool_shed/galaxy_install/tool_migration_manager.py
+1 -1
View File
@@ -581,7 +581,7 @@ class ToolMigrationManager( object ):
owner=self.repository_owner,
changeset_revision=tool_shed_repository.installed_changeset_revision )
pathspec = [ 'repository', 'get_tool_versions' ]
text = common_util.tool_shed_get( self.app, self.tool_shed_url, pathspec=pathspec, params=params )
text = util.url_get( self.tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( self.tool_shed_url ), pathspec=pathspec, params=params )
if text:
tool_version_dicts = json.loads( text )
tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository )
lib/tool_shed/galaxy_install/update_repository_manager.py
+2 -1
View File
@@ -9,6 +9,7 @@ from sqlalchemy import false
import tool_shed.util.shed_util_common as suc
from tool_shed.util import common_util
from tool_shed.util import encoding_util
from galaxy import util
log = logging.getLogger( __name__ )
@@ -36,7 +37,7 @@ class UpdateRepositoryManager( object ):
changeset_revision=str( repository.installed_changeset_revision ) )
pathspec = [ 'repository', 'get_changeset_revision_and_ctx_rev' ]
try:
encoded_update_dict = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params )
encoded_update_dict = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if encoded_update_dict:
update_dict = encoding_util.tool_shed_decode( encoded_update_dict )
includes_data_managers = update_dict.get( 'includes_data_managers', False )
lib/tool_shed/scripts/api/common.py
+3 -2
View File
@@ -6,7 +6,8 @@ import urllib2
sys.path.insert(1, os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir, os.pardir ) )
from tool_shed.util import common_util, hg_util
from tool_shed.util import hg_util
from galaxy import util
class HTTPRedirectWithDataHandler( urllib2.HTTPRedirectHandler ):
@@ -131,7 +132,7 @@ def get_api_url( base, parts=[], params=None ):
parts.insert( 0, 'api' )
elif 'api' not in parts:
parts.insert( 0, 'api' )
url = common_util.url_join( base, pathspec=parts, params=params )
url = util.build_url( base, pathspec=parts, params=params )
return url
lib/tool_shed/scripts/deprecate_repositories_without_metadata.py
+2 -2
View File
@@ -19,7 +19,7 @@ from sqlalchemy import and_, distinct, false, not_
import galaxy.webapps.tool_shed.config as tool_shed_config
import galaxy.webapps.tool_shed.model.mapping
from galaxy.util import send_mail as galaxy_send_mail
from tool_shed.util.common_util import url_join
from galaxy.util import build_url
log = logging.getLogger()
log.setLevel( 10 )
@@ -28,7 +28,7 @@ assert sys.version_info[:2] >= ( 2, 4 )
def build_citable_url( host, repository ):
return url_join( host, pathspec=[ 'view', repository.user.username, repository.name ] )
return build_url( host, pathspec=[ 'view', repository.user.username, repository.name ] )
def main():
lib/tool_shed/tool_shed_registry.py
+8 -3
View File
@@ -1,8 +1,8 @@
import logging
import urllib2
from galaxy.util.odict import odict
from tool_shed.util import common_util, xml_util
from six.moves.urllib import request as urlrequest
log = logging.getLogger( __name__ )
@@ -31,7 +31,7 @@ class Registry( object ):
self.tool_sheds_auth[ name ] = None
log.debug( 'Loaded reference to tool shed: %s' % name )
if name and url and username and password:
pass_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
pass_mgr = urlrequest.HTTPPasswordMgrWithDefaultRealm()
pass_mgr.add_password( None, url, username, password )
self.tool_sheds_auth[ name ] = pass_mgr
except Exception, e:
@@ -40,7 +40,7 @@ class Registry( object ):
def password_manager_for_url( self, url ):
"""
If the tool shed is using external auth, the client to the tool shed must authenticate to that
as well. This provides access to the urllib2.HTTPPasswordMgrWithdefaultRealm() object for the
as well. This provides access to the six.moves.urllib.request.HTTPPasswordMgrWithdefaultRealm() object for the
url passed in.
Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at
@@ -53,3 +53,8 @@ class Registry( object ):
return self.tool_sheds_auth[ shed_name ]
log.debug( "Invalid url '%s' received by tool shed registry's password_manager_for_url method." % str( url ) )
return None
def url_auth( self, url ):
password_manager = self.password_manager_for_url( url )
if password_manager is not None:
return urlrequest.HTTPBasicAuthHandler( password_manager )
lib/tool_shed/util/common_util.py
+6 -40
View File
@@ -1,9 +1,6 @@
import json
import logging
import os
import urllib
import urllib2
from six import string_types
from galaxy import util
from galaxy.util.odict import odict
@@ -114,7 +111,7 @@ def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_d
def generate_clone_url_for_installed_repository( app, repository ):
"""Generate the URL for cloning a repository that has been installed into a Galaxy instance."""
tool_shed_url = get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) )
return url_join( tool_shed_url, pathspec=[ 'repos', str( repository.owner ), str( repository.name ) ] )
return util.build_url( tool_shed_url, pathspec=[ 'repos', str( repository.owner ), str( repository.name ) ] )
def generate_clone_url_for_repository_in_tool_shed( user, repository ):
@@ -135,7 +132,7 @@ def generate_clone_url_from_repo_info_tup( app, repo_info_tup ):
parse_repository_dependency_tuple( repo_info_tup )
tool_shed_url = get_tool_shed_url_from_tool_shed_registry( app, toolshed )
# Don't include the changeset_revision in clone urls.
return url_join( tool_shed_url, pathspec=[ 'repos', owner, name ] )
return util.build_url( tool_shed_url, pathspec=[ 'repos', owner, name ] )
def get_non_shed_tool_panel_configs( app ):
@@ -160,11 +157,11 @@ def get_repository_dependencies( app, tool_shed_url, repository_name, repository
params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_repository_dependencies' ]
try:
raw_text = tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
tool_shed_accessible = True
except Exception, e:
tool_shed_accessible = False
log.warn( "The URL\n%s\nraised the exception:\n%s\n", url_join( tool_shed_url, pathspec=pathspec, params=params ), e )
log.warn( "The URL\n%s\nraised the exception:\n%s\n", util.build_url( tool_shed_url, pathspec=pathspec, params=params ), e )
if tool_shed_accessible:
if len( raw_text ) > 2:
encoded_text = json.loads( raw_text )
@@ -192,11 +189,11 @@ def get_tool_dependencies( app, tool_shed_url, repository_name, repository_owner
params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_tool_dependencies' ]
try:
text = tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
tool_shed_accessible = True
except Exception, e:
tool_shed_accessible = False
log.warn( "The URL\n%s\nraised the exception:\n%s\n", url_join( tool_shed_url, pathspec=pathspec, params=params ), e )
log.warn( "The URL\n%s\nraised the exception:\n%s\n", util.build_url( tool_shed_url, pathspec=pathspec, params=params ), e )
if tool_shed_accessible:
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode( text )
@@ -339,34 +336,3 @@ def remove_protocol_and_user_from_clone_url( repository_clone_url ):
def remove_protocol_from_tool_shed_url( tool_shed_url ):
"""Return a partial Tool Shed URL, eliminating the protocol if it exists."""
return util.remove_protocol_from_url( tool_shed_url )
def tool_shed_get( app, base_url, pathspec=None, params=None ):
"""Make contact with the tool shed via the uri provided."""
registry = app.tool_shed_registry
# urllib2 auto-detects system proxies, when passed a Proxyhandler.
# Refer: https://docs.python.org/2/howto/urllib2.html#proxies
proxy = urllib2.ProxyHandler()
urlopener = urllib2.build_opener( proxy )
urllib2.install_opener( urlopener )
password_mgr = registry.password_manager_for_url( base_url )
if password_mgr is not None:
auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr )
urlopener.add_handler( auth_handler )
full_url = url_join( base_url, pathspec=pathspec, params=params )
response = urlopener.open( full_url )
content = response.read()
response.close()
return content
def url_join( base_url, pathspec=None, params=None ):
"""Return a valid URL produced by appending a base URL and a set of request parameters."""
url = base_url.rstrip( '/' )
if pathspec is not None:
if not isinstance( pathspec, string_types ):
pathspec = '/'.join( pathspec )
url = '%s/%s' % ( url, pathspec )
if params is not None:
url = '%s?%s' % ( url, urllib.urlencode( params ) )
return url
lib/tool_shed/util/readme_util.py
+2 -2
View File
@@ -7,7 +7,7 @@ from mako.template import Template
import tool_shed.util.shed_util_common as suc
from galaxy import web
from galaxy.util import rst_to_html, unicodify
from galaxy.util import rst_to_html, unicodify, url_get
from tool_shed.util import basic_util, common_util, hg_util
log = logging.getLogger( __name__ )
@@ -97,7 +97,7 @@ def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_readme_files' ]
raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
raw_text = url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
readme_files_dict = json.loads( raw_text )
return readme_files_dict
lib/tool_shed/util/shed_util_common.py
+9 -9
View File
@@ -254,7 +254,7 @@ def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_r
tool_shed_url = items[ 0 ]
repo_path = items[ 1 ]
tool_shed_url = common_util.remove_port_from_tool_shed_url( tool_shed_url )
return common_util.url_join( tool_shed_url, pathspec=[ 'repos', repo_path, changeset_revision ] )
return '/'.join( [ tool_shed_url, 'repos', repo_path, changeset_revision ] )
def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ):
@@ -301,7 +301,7 @@ def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_ctx_rev' ]
ctx_rev = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
ctx_rev = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return ctx_rev
@@ -369,7 +369,7 @@ def get_tool_dependency_definition_metadata_from_tool_shed( app, tool_shed_url,
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=owner )
pathspec = [ 'repository', 'get_tool_dependency_definition_metadata' ]
metadata = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
metadata = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return metadata
@@ -626,7 +626,7 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed )
params = dict( name=name, owner=owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'next_installable_changeset_revision' ]
text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
repository = get_installed_repository( app=app,
tool_shed=tool_shed,
@@ -828,7 +828,7 @@ def get_repository_type_from_tool_shed( app, tool_shed_url, name, owner ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=owner )
pathspec = [ 'repository', 'get_repository_type' ]
repository_type = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
repository_type = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return repository_type
@@ -914,7 +914,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ):
params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision )
pathspec = [ 'repository', 'status_for_installed_repository' ]
try:
encoded_tool_shed_status_dict = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
encoded_tool_shed_status_dict = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
tool_shed_status_dict = encoding_util.tool_shed_decode( encoded_tool_shed_status_dict )
return tool_shed_status_dict
except HTTPError, e:
@@ -926,7 +926,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ):
params[ 'from_update_manager' ] = True
try:
# The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision.
text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return dict( revision_update=text )
except Exception, e:
# The required tool shed may be unavailable, so default the revision_update value to 'false'.
@@ -1021,7 +1021,7 @@ def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, ow
raise Exception( message )
params = dict( name=name, owner=owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'updated_changeset_revisions' ]
text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return text
@@ -1227,7 +1227,7 @@ def repository_was_previously_installed( app, tool_shed_url, repository_name, re
changeset_revision=changeset_revision,
from_tip=str( from_tip ) )
pathspec = [ 'repository', 'previous_changeset_revisions' ]
text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
changeset_revisions = util.listify( text )
for previous_changeset_revision in changeset_revisions: