diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py
index be904c17afc..3478801d726 100644
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -30,6 +30,7 @@ from xml.etree import ElementInclude, ElementTree
from six import binary_type, iteritems, PY3, string_types, text_type
from six.moves import email_mime_text, xrange, zip
from six.moves.urllib import parse as urlparse
+from six.moves.urllib import request as urlrequest
try:
import docutils.core as docutils_core
@@ -1342,6 +1343,46 @@ def parse_int(value, min_val=None, max_val=None, default=None, allow_none=False)
raise
+def build_url( base_url, port=80, scheme='http', pathspec=None, params=None, doseq=False ):
+ if params is None:
+ params = dict()
+ if pathspec is None:
+ pathspec = []
+ parsed_url = urlparse.urlparse( base_url )
+ if scheme != 'http':
+ parsed_url.scheme = scheme
+ if port != 80:
+ url = '%s://%s:%d/%s' % ( parsed_url.scheme, parsed_url.netloc.rstrip( '/' ), int( port ), parsed_url.path )
+ else:
+ url = '%s://%s/%s' % ( parsed_url.scheme, parsed_url.netloc.rstrip( '/' ), parsed_url.path.lstrip( '/' ) )
+ if len( pathspec ) > 0:
+ url = '%s/%s' % ( url.rstrip( '/' ), '/'.join( pathspec ) )
+ if parsed_url.query:
+ for query_parameter in parsed_url.query.split( '&' ):
+ key, value = query_parameter.split( '=' )
+ params[ key ] = value
+ if params:
+ url += '?%s' % urlparse.urlencode( params, doseq=doseq )
+ return url
+
+
+def url_get( base_url, password_mgr=None, pathspec=None, params=None ):
+ """Make contact with the uri provided and return any contents."""
+ # Uses system proxy settings if they exist.
+ proxy = urlrequest.ProxyHandler()
+ if password_mgr is not None:
+ auth = urlrequest.HTTPDigestAuthHandler( password_mgr )
+ urlopener = urlrequest.build_opener( proxy, auth )
+ else:
+ urlopener = urlrequest.build_opener( proxy )
+ urlrequest.install_opener( urlopener )
+ full_url = build_url( base_url, pathspec=pathspec, params=params )
+ response = urlopener.open( full_url )
+ content = response.read()
+ response.close()
+ return content
+
+
def safe_relpath(path):
"""
Given what we expect to be a relative path, determine whether the path
diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py
index 215cd0b7b3c..d771b998f58 100644
--- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py
+++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py
@@ -13,7 +13,6 @@ from galaxy.web.base.controller import BaseAPIController
from tool_shed.galaxy_install.install_manager import InstallRepositoryManager
from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager
-from tool_shed.util import common_util
from tool_shed.util import encoding_util
from tool_shed.util import hg_util
from tool_shed.util import workflow_util
@@ -100,7 +99,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
params = dict(name=name, owner=owner)
pathspec = ['api', 'repositories', 'get_ordered_installable_revisions']
try:
- raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec, params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
except Exception, e:
message = "Error attempting to retrieve the latest installable revision from tool shed %s for repository %s owned by %s: %s" % \
( str( tool_shed_url ), str( name ), str( owner ), str( e ) )
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py
index 1f6109fb759..d3c1414c549 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin.py
@@ -11,7 +11,7 @@ from galaxy.actions.admin import AdminActions
from galaxy.exceptions import MessageException
from galaxy.model import tool_shed_install as install_model
from galaxy.model.util import pgcalc
-from galaxy.util import nice_size, sanitize_text
+from galaxy.util import nice_size, sanitize_text, url_get
from galaxy.util.odict import odict
from galaxy.web import url_for
from galaxy.web.base.controller import BaseUIController, UsesQuotaMixin
@@ -770,9 +770,9 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
tree = galaxy.util.parse_xml( tools_xml_file_path )
root = tree.getroot()
tool_shed = root.get( 'name' )
- tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed )
+ shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed )
repo_name_dependency_tups = []
- if tool_shed_url:
+ if shed_url:
for elem in root:
if elem.tag == 'repository':
tool_dependencies = []
@@ -781,7 +781,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
changeset_revision = elem.get( 'changeset_revision' )
params = dict( name=repository_name, owner='devteam', changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_tool_dependencies' ]
- text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+ text = url_get( shed_url, password_mgr=self.app.tool_shed_registry.url_auth( shed_url ), pathspec=pathspec, params=params )
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode( text )
for dependency_key, requirements_dict in tool_dependencies_dict.items():
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
index 4ca6110d570..624a26069df 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
@@ -189,7 +189,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = kwd.get( 'tool_shed_url', '' )
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
- url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params )
+ url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -212,7 +212,7 @@ class AdminToolshed( AdminGalaxy ):
owner=str( repository.owner ),
changeset_revision=str( repository.changeset_revision ) )
pathspec = [ 'repository', 'check_for_updates' ]
- url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+ url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -369,7 +369,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = kwd.get( 'tool_shed_url', '' )
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
- url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'find_tools' ], params=params )
+ url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'find_tools' ], params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -378,7 +378,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = kwd.get( 'tool_shed_url', '' )
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
params = dict( galaxy_url=web.url_for( '/', qualified=True ) )
- url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'find_workflows' ], params=params )
+ url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'find_workflows' ], params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -413,7 +413,7 @@ class AdminToolshed( AdminGalaxy ):
raise Exception( message )
params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_tool_dependencies' ]
- raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if len( raw_text ) > 2:
encoded_text = json.loads( raw_text )
text = encoding_util.tool_shed_decode( encoded_text )
@@ -439,7 +439,7 @@ class AdminToolshed( AdminGalaxy ):
owner=str( repository_owner ),
changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_updated_repository_information' ]
- raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
repo_information_dict = json.loads( raw_text )
return repo_information_dict
@@ -529,8 +529,8 @@ class AdminToolshed( AdminGalaxy ):
name=name,
owner=owner )
pathspec = [ 'repository', 'get_latest_downloadable_changeset_revision' ]
- raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
- url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
+ url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
latest_downloadable_revision = json.loads( raw_text )
if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH:
message = 'Error retrieving the latest downloadable revision for this repository via the url %s.' % url
@@ -541,7 +541,7 @@ class AdminToolshed( AdminGalaxy ):
# appropriate repository revision if one exists. We need to create a temporary repo_info_tuple
# with the following entries to handle this.
# ( description, clone_url, changeset_revision, ctx_rev, owner, repository_dependencies, tool_dependencies )
- tmp_clone_url = common_util.url_join( tool_shed_url, pathspec=[ 'repos', owner, name ] )
+ tmp_clone_url = util.build_url( tool_shed_url, pathspec=[ 'repos', owner, name ] )
tmp_repo_info_tuple = ( None, tmp_clone_url, latest_downloadable_revision, None, owner, None, None )
installed_repository, installed_changeset_revision = \
suc.repository_was_previously_installed( trans.app, tool_shed_url, name, tmp_repo_info_tuple, from_tip=False )
@@ -559,7 +559,7 @@ class AdminToolshed( AdminGalaxy ):
changeset_revisions=str( latest_downloadable_revision ),
galaxy_url=web.url_for( '/', qualified=True ) )
pathspec = [ 'repository', 'install_repositories_by_revision' ]
- url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+ url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
else:
message = 'Cannot locate installed tool shed repository with encoded id %s.' % str( repository_id )
@@ -727,7 +727,7 @@ class AdminToolshed( AdminGalaxy ):
changeset_revisions=installed_changeset_revision,
galaxy_url=web.url_for( '/', qualified=True ) )
pathspec = [ 'repository', 'install_repositories_by_revision' ]
- url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+ url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
description = kwd.get( 'description', repository.description )
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository )
@@ -985,7 +985,7 @@ class AdminToolshed( AdminGalaxy ):
try:
params = dict( name=str( repository.name ), owner=str( repository.owner ) )
pathspec = [ 'repository', 'get_repository_id' ]
- repository_ids = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+ repository_ids = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
except Exception, e:
# The Tool Shed cannot handle the get_repository_id request, so the code must be older than the
# 04/2014 Galaxy release when it was introduced. It will be safest to error out and let the
@@ -1005,7 +1005,7 @@ class AdminToolshed( AdminGalaxy ):
# Get the information necessary to install each repository.
params = dict( repository_ids=str( repository_ids ), changeset_revisions=str( changeset_revisions ) )
pathspec = [ 'repository', 'get_repository_information' ]
- raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
repo_information_dict = json.loads( raw_text )
for encoded_repo_info_dict in repo_information_dict.get( 'repo_info_dicts', [] ):
decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict )
@@ -1539,7 +1539,7 @@ class AdminToolshed( AdminGalaxy ):
owner=tool_shed_repository.owner,
changeset_revision=tool_shed_repository.installed_changeset_revision )
pathspec = [ 'repository', 'get_readme_files' ]
- raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
readme_files_dict = json.loads( raw_text )
tool_dependencies = metadata.get( 'tool_dependencies', None )
rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app )
@@ -1739,7 +1739,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) )
params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision )
pathspec = [ 'repository', 'get_tool_versions' ]
- text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+ text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
tool_version_dicts = json.loads( text )
tvm = tool_version_manager.ToolVersionManager( trans.app )
diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py
index c92a5fbf878..7fcfa825131 100644
--- a/lib/galaxy/webapps/galaxy/controllers/workflow.py
+++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py
@@ -10,7 +10,6 @@ from sqlalchemy import and_
from sqlalchemy.sql import expression
from markupsafe import escape
-from tool_shed.util import common_util
from tool_shed.util import encoding_util
from galaxy import model
@@ -862,7 +861,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
workflow_name=encoding_util.tool_shed_encode( workflow_name ),
open_for_url=True )
pathspec = [ 'workflow', 'import_workflow' ]
- workflow_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params )
+ workflow_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
import_button = True
if import_button:
workflow_data = None
diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py
index df8530c33f0..646d470f823 100644
--- a/lib/galaxy/webapps/tool_shed/controllers/repository.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py
@@ -809,7 +809,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
return update
params['latest_changeset_revision'] = str( latest_changeset_revision )
params['latest_ctx_rev'] = str( update_to_ctx.rev() )
- url = common_util.url_join( galaxy_url, pathspec=pathspec, params=params )
+ url = util.build_url( galaxy_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
@web.expose
@@ -1065,7 +1065,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
trans.sa_session.flush()
tool_shed_url = web.url_for( '/', qualified=True )
pathspec = [ 'repos', str( repository.user.username ), str( repository.name ), 'archive', file_type_str ]
- download_url = common_util.url_join( tool_shed_url, pathspec=pathspec )
+ download_url = util.build_url( tool_shed_url, pathspec=pathspec )
return trans.response.send_redirect( download_url )
@web.expose
@@ -1534,7 +1534,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
time_tested = repository_metadata.time_last_tested.strftime( '%a, %d %b %Y %H:%M:%S UT' )
# Generate a citable URL for this repository with owner and changeset revision.
pathspec = [ 'view', str( user.username ), str( repository.name ), str( repository_metadata.changeset_revision ) ]
- repository_citable_url = common_util.url_join( tool_shed_url, pathspec=pathspec )
+ repository_citable_url = util.build_url( tool_shed_url, pathspec=pathspec )
passed_tests = len( tool_test_results.get( 'passed_tests', [] ) )
failed_tests = len( tool_test_results.get( 'failed_tests', [] ) )
missing_test_components = len( tool_test_results.get( 'missing_test_components', [] ) )
@@ -1982,7 +1982,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
repository_ids=','.join( util.listify( repository_ids ) ),
changeset_revisions=','.join( util.listify( changeset_revisions ) ) )
pathspec = [ 'admin_toolshed', 'prepare_for_install' ]
- url = common_util.url_join( galaxy_url, pathspec=pathspec, params=params )
+ url = util.build_url( galaxy_url, pathspec=pathspec, params=params )
return trans.response.send_redirect( url )
else:
message = 'Repository installation is not possible due to an invalid Galaxy URL: %s. ' % galaxy_url
diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py
index 7dd879bf6eb..5fcd84c5b18 100644
--- a/lib/tool_shed/capsule/capsule_manager.py
+++ b/lib/tool_shed/capsule/capsule_manager.py
@@ -15,6 +15,7 @@ from galaxy import web
from galaxy.util import asbool
from galaxy.util import CHUNK_SIZE
from galaxy.util import safe_relpath
+from galaxy.util import build_url
from galaxy.util.odict import odict
from tool_shed.dependencies.repository.relation_builder import RelationBuilder
from tool_shed.dependencies import attribute_handlers
@@ -123,7 +124,7 @@ class ExportRepositoryManager( object ):
params = dict( encoded_repositories_archive_name=encoded_repositories_archive_name )
pathspec = [ 'repository', 'export_via_api' ]
tool_shed_url = web.url_for( '/', qualified=True )
- download_url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
+ download_url = build_url( tool_shed_url, pathspec=pathspec, params=params )
return dict( download_url=download_url, error_messages=error_messages )
return repositories_archive, error_messages
diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py
index 3b92497f521..b5600bf3c66 100644
--- a/lib/tool_shed/galaxy_install/dependency_display.py
+++ b/lib/tool_shed/galaxy_install/dependency_display.py
@@ -455,7 +455,7 @@ class DependencyDisplayer( object ):
owner=str( repository.owner ),
changeset_revision=str( repository.installed_changeset_revision ) )
pathspec = [ 'repository', 'get_readme_files' ]
- raw_text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
readme_files_dict = json.loads( raw_text )
else:
readme_files_dict = readme_util.build_readme_files_dict( self.app,
diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py
index 8351a935777..a03d8e72979 100644
--- a/lib/tool_shed/galaxy_install/install_manager.py
+++ b/lib/tool_shed/galaxy_install/install_manager.py
@@ -460,7 +460,7 @@ class InstallRepositoryManager( object ):
changeset_revision=str( changeset_revision ) )
pathspec = [ 'api', 'repositories', 'get_repository_revision_install_info' ]
try:
- raw_text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
except Exception, e:
message = "Error attempting to retrieve installation information from tool shed "
message += "%s for revision %s of repository %s owned by %s: %s" % \
@@ -1005,8 +1005,8 @@ def fetch_tool_versions( app, tool_shed_repository ):
owner=str( tool_shed_repository.owner ),
changeset_revision=str( tool_shed_repository.changeset_revision ) )
pathspec = [ 'repository', 'get_tool_versions' ]
- url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params )
- text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ url = util.build_url( tool_shed_url, pathspec=pathspec, params=params )
+ text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
return json.loads( text )
else:
diff --git a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py
index b03417da16f..a69c0357aec 100644
--- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py
+++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py
@@ -8,7 +8,7 @@ import os
import urllib
import urllib2
-from galaxy.util import asbool
+from galaxy.util import asbool, url_get, build_url
from tool_shed.galaxy_install.tools import tool_panel_manager
@@ -291,9 +291,9 @@ class RepositoryDependencyInstallManager( object ):
changeset_revision=str( repository.changeset_revision ) )
pathspec = [ 'repository', 'get_repository_dependencies' ]
try:
- raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
except Exception, e:
- log.error("The URL\n%s\nraised the exception:\n%s\n", common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ), str( e ) )
+ log.error("The URL\n%s\nraised the exception:\n%s\n", build_url( tool_shed_url, pathspec=pathspec, params=params ), str( e ) )
return ''
if len( raw_text ) > 2:
encoded_text = json.loads( raw_text )
@@ -378,7 +378,7 @@ class RepositoryDependencyInstallManager( object ):
# Handle secure / insecure Tool Shed URL protocol changes and port changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
pathspec = [ 'repository', 'get_required_repo_info_dict' ]
- url = common_util.url_join( tool_shed_url, pathspec=pathspec )
+ url = build_url( tool_shed_url, pathspec=pathspec )
# Fix for handling 307 redirect not being handled nicely by urllib2.urlopen when the urllib2.Request has data provided
url = urllib2.urlopen( urllib2.Request( url ) ).geturl()
request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
index 2198ceb8512..d505325a8be 100644
--- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
+++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
@@ -3,7 +3,7 @@ import os
import tempfile
from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY
-from galaxy.util import listify
+from galaxy.util import listify, url_get
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import shed_util_common as suc
@@ -249,7 +249,7 @@ class Repository( RecipeTag, SyncDatabase ):
owner=owner,
changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_tool_dependencies_config_contents' ]
- text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params )
+ text = url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
# Write the contents to a temporary file on disk so it can be reloaded and parsed.
fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-cttdc" )
diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py
index 64842c07d1f..104a0d84547 100644
--- a/lib/tool_shed/galaxy_install/tool_migration_manager.py
+++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py
@@ -581,7 +581,7 @@ class ToolMigrationManager( object ):
owner=self.repository_owner,
changeset_revision=tool_shed_repository.installed_changeset_revision )
pathspec = [ 'repository', 'get_tool_versions' ]
- text = common_util.tool_shed_get( self.app, self.tool_shed_url, pathspec=pathspec, params=params )
+ text = util.url_get( self.tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( self.tool_shed_url ), pathspec=pathspec, params=params )
if text:
tool_version_dicts = json.loads( text )
tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository )
diff --git a/lib/tool_shed/galaxy_install/update_repository_manager.py b/lib/tool_shed/galaxy_install/update_repository_manager.py
index 824f5237e10..aa8e60e9b61 100644
--- a/lib/tool_shed/galaxy_install/update_repository_manager.py
+++ b/lib/tool_shed/galaxy_install/update_repository_manager.py
@@ -9,6 +9,7 @@ from sqlalchemy import false
import tool_shed.util.shed_util_common as suc
from tool_shed.util import common_util
from tool_shed.util import encoding_util
+from galaxy import util
log = logging.getLogger( __name__ )
@@ -36,7 +37,7 @@ class UpdateRepositoryManager( object ):
changeset_revision=str( repository.installed_changeset_revision ) )
pathspec = [ 'repository', 'get_changeset_revision_and_ctx_rev' ]
try:
- encoded_update_dict = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params )
+ encoded_update_dict = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if encoded_update_dict:
update_dict = encoding_util.tool_shed_decode( encoded_update_dict )
includes_data_managers = update_dict.get( 'includes_data_managers', False )
diff --git a/lib/tool_shed/scripts/api/common.py b/lib/tool_shed/scripts/api/common.py
index 02e2b3b1a6a..88f7fcf2e94 100644
--- a/lib/tool_shed/scripts/api/common.py
+++ b/lib/tool_shed/scripts/api/common.py
@@ -6,7 +6,8 @@ import urllib2
sys.path.insert(1, os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir, os.pardir ) )
-from tool_shed.util import common_util, hg_util
+from tool_shed.util import hg_util
+from galaxy import util
class HTTPRedirectWithDataHandler( urllib2.HTTPRedirectHandler ):
@@ -131,7 +132,7 @@ def get_api_url( base, parts=[], params=None ):
parts.insert( 0, 'api' )
elif 'api' not in parts:
parts.insert( 0, 'api' )
- url = common_util.url_join( base, pathspec=parts, params=params )
+ url = util.build_url( base, pathspec=parts, params=params )
return url
diff --git a/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py b/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py
index ed86ce60be8..3f090f2edba 100644
--- a/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py
+++ b/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py
@@ -19,7 +19,7 @@ from sqlalchemy import and_, distinct, false, not_
import galaxy.webapps.tool_shed.config as tool_shed_config
import galaxy.webapps.tool_shed.model.mapping
from galaxy.util import send_mail as galaxy_send_mail
-from tool_shed.util.common_util import url_join
+from galaxy.util import build_url
log = logging.getLogger()
log.setLevel( 10 )
@@ -28,7 +28,7 @@ assert sys.version_info[:2] >= ( 2, 4 )
def build_citable_url( host, repository ):
- return url_join( host, pathspec=[ 'view', repository.user.username, repository.name ] )
+ return build_url( host, pathspec=[ 'view', repository.user.username, repository.name ] )
def main():
diff --git a/lib/tool_shed/tool_shed_registry.py b/lib/tool_shed/tool_shed_registry.py
index 5b26bf62395..6014ee5b414 100644
--- a/lib/tool_shed/tool_shed_registry.py
+++ b/lib/tool_shed/tool_shed_registry.py
@@ -1,8 +1,8 @@
import logging
-import urllib2
from galaxy.util.odict import odict
from tool_shed.util import common_util, xml_util
+from six.moves.urllib import request as urlrequest
log = logging.getLogger( __name__ )
@@ -31,7 +31,7 @@ class Registry( object ):
self.tool_sheds_auth[ name ] = None
log.debug( 'Loaded reference to tool shed: %s' % name )
if name and url and username and password:
- pass_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ pass_mgr = urlrequest.HTTPPasswordMgrWithDefaultRealm()
pass_mgr.add_password( None, url, username, password )
self.tool_sheds_auth[ name ] = pass_mgr
except Exception, e:
@@ -40,7 +40,7 @@ class Registry( object ):
def password_manager_for_url( self, url ):
"""
If the tool shed is using external auth, the client to the tool shed must authenticate to that
- as well. This provides access to the urllib2.HTTPPasswordMgrWithdefaultRealm() object for the
+ as well. This provides access to the six.moves.urllib.request.HTTPPasswordMgrWithdefaultRealm() object for the
url passed in.
Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at
@@ -53,3 +53,8 @@ class Registry( object ):
return self.tool_sheds_auth[ shed_name ]
log.debug( "Invalid url '%s' received by tool shed registry's password_manager_for_url method." % str( url ) )
return None
+
+ def url_auth( self, url ):
+ password_manager = self.password_manager_for_url( url )
+ if password_manager is not None:
+ return urlrequest.HTTPBasicAuthHandler( password_manager )
diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py
index a3d5f6561ce..2db00a001ea 100644
--- a/lib/tool_shed/util/common_util.py
+++ b/lib/tool_shed/util/common_util.py
@@ -1,9 +1,6 @@
import json
import logging
import os
-import urllib
-import urllib2
-from six import string_types
from galaxy import util
from galaxy.util.odict import odict
@@ -114,7 +111,7 @@ def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_d
def generate_clone_url_for_installed_repository( app, repository ):
"""Generate the URL for cloning a repository that has been installed into a Galaxy instance."""
tool_shed_url = get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) )
- return url_join( tool_shed_url, pathspec=[ 'repos', str( repository.owner ), str( repository.name ) ] )
+ return util.build_url( tool_shed_url, pathspec=[ 'repos', str( repository.owner ), str( repository.name ) ] )
def generate_clone_url_for_repository_in_tool_shed( user, repository ):
@@ -135,7 +132,7 @@ def generate_clone_url_from_repo_info_tup( app, repo_info_tup ):
parse_repository_dependency_tuple( repo_info_tup )
tool_shed_url = get_tool_shed_url_from_tool_shed_registry( app, toolshed )
# Don't include the changeset_revision in clone urls.
- return url_join( tool_shed_url, pathspec=[ 'repos', owner, name ] )
+ return util.build_url( tool_shed_url, pathspec=[ 'repos', owner, name ] )
def get_non_shed_tool_panel_configs( app ):
@@ -160,11 +157,11 @@ def get_repository_dependencies( app, tool_shed_url, repository_name, repository
params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_repository_dependencies' ]
try:
- raw_text = tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
tool_shed_accessible = True
except Exception, e:
tool_shed_accessible = False
- log.warn( "The URL\n%s\nraised the exception:\n%s\n", url_join( tool_shed_url, pathspec=pathspec, params=params ), e )
+ log.warn( "The URL\n%s\nraised the exception:\n%s\n", util.build_url( tool_shed_url, pathspec=pathspec, params=params ), e )
if tool_shed_accessible:
if len( raw_text ) > 2:
encoded_text = json.loads( raw_text )
@@ -192,11 +189,11 @@ def get_tool_dependencies( app, tool_shed_url, repository_name, repository_owner
params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_tool_dependencies' ]
try:
- text = tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
tool_shed_accessible = True
except Exception, e:
tool_shed_accessible = False
- log.warn( "The URL\n%s\nraised the exception:\n%s\n", url_join( tool_shed_url, pathspec=pathspec, params=params ), e )
+ log.warn( "The URL\n%s\nraised the exception:\n%s\n", util.build_url( tool_shed_url, pathspec=pathspec, params=params ), e )
if tool_shed_accessible:
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode( text )
@@ -339,34 +336,3 @@ def remove_protocol_and_user_from_clone_url( repository_clone_url ):
def remove_protocol_from_tool_shed_url( tool_shed_url ):
"""Return a partial Tool Shed URL, eliminating the protocol if it exists."""
return util.remove_protocol_from_url( tool_shed_url )
-
-
-def tool_shed_get( app, base_url, pathspec=None, params=None ):
- """Make contact with the tool shed via the uri provided."""
- registry = app.tool_shed_registry
- # urllib2 auto-detects system proxies, when passed a Proxyhandler.
- # Refer: https://docs.python.org/2/howto/urllib2.html#proxies
- proxy = urllib2.ProxyHandler()
- urlopener = urllib2.build_opener( proxy )
- urllib2.install_opener( urlopener )
- password_mgr = registry.password_manager_for_url( base_url )
- if password_mgr is not None:
- auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr )
- urlopener.add_handler( auth_handler )
- full_url = url_join( base_url, pathspec=pathspec, params=params )
- response = urlopener.open( full_url )
- content = response.read()
- response.close()
- return content
-
-
-def url_join( base_url, pathspec=None, params=None ):
- """Return a valid URL produced by appending a base URL and a set of request parameters."""
- url = base_url.rstrip( '/' )
- if pathspec is not None:
- if not isinstance( pathspec, string_types ):
- pathspec = '/'.join( pathspec )
- url = '%s/%s' % ( url, pathspec )
- if params is not None:
- url = '%s?%s' % ( url, urllib.urlencode( params ) )
- return url
diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py
index a68f01f1036..82b919db440 100644
--- a/lib/tool_shed/util/readme_util.py
+++ b/lib/tool_shed/util/readme_util.py
@@ -7,7 +7,7 @@ from mako.template import Template
import tool_shed.util.shed_util_common as suc
from galaxy import web
-from galaxy.util import rst_to_html, unicodify
+from galaxy.util import rst_to_html, unicodify, url_get
from tool_shed.util import basic_util, common_util, hg_util
log = logging.getLogger( __name__ )
@@ -97,7 +97,7 @@ def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_readme_files' ]
- raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ raw_text = url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
readme_files_dict = json.loads( raw_text )
return readme_files_dict
diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py
index 9596585ec40..5c0fefa7e63 100644
--- a/lib/tool_shed/util/shed_util_common.py
+++ b/lib/tool_shed/util/shed_util_common.py
@@ -254,7 +254,7 @@ def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_r
tool_shed_url = items[ 0 ]
repo_path = items[ 1 ]
tool_shed_url = common_util.remove_port_from_tool_shed_url( tool_shed_url )
- return common_util.url_join( tool_shed_url, pathspec=[ 'repos', repo_path, changeset_revision ] )
+ return '/'.join( [ tool_shed_url, 'repos', repo_path, changeset_revision ] )
def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ):
@@ -301,7 +301,7 @@ def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_ctx_rev' ]
- ctx_rev = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ ctx_rev = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return ctx_rev
@@ -369,7 +369,7 @@ def get_tool_dependency_definition_metadata_from_tool_shed( app, tool_shed_url,
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=owner )
pathspec = [ 'repository', 'get_tool_dependency_definition_metadata' ]
- metadata = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ metadata = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return metadata
@@ -626,7 +626,7 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed )
params = dict( name=name, owner=owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'next_installable_changeset_revision' ]
- text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
repository = get_installed_repository( app=app,
tool_shed=tool_shed,
@@ -828,7 +828,7 @@ def get_repository_type_from_tool_shed( app, tool_shed_url, name, owner ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=owner )
pathspec = [ 'repository', 'get_repository_type' ]
- repository_type = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ repository_type = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return repository_type
@@ -914,7 +914,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ):
params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision )
pathspec = [ 'repository', 'status_for_installed_repository' ]
try:
- encoded_tool_shed_status_dict = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ encoded_tool_shed_status_dict = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
tool_shed_status_dict = encoding_util.tool_shed_decode( encoded_tool_shed_status_dict )
return tool_shed_status_dict
except HTTPError, e:
@@ -926,7 +926,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ):
params[ 'from_update_manager' ] = True
try:
# The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision.
- text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return dict( revision_update=text )
except Exception, e:
# The required tool shed may be unavailable, so default the revision_update value to 'false'.
@@ -1021,7 +1021,7 @@ def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, ow
raise Exception( message )
params = dict( name=name, owner=owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'updated_changeset_revisions' ]
- text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
return text
@@ -1227,7 +1227,7 @@ def repository_was_previously_installed( app, tool_shed_url, repository_name, re
changeset_revision=changeset_revision,
from_tip=str( from_tip ) )
pathspec = [ 'repository', 'previous_changeset_revisions' ]
- text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+ text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
if text:
changeset_revisions = util.listify( text )
for previous_changeset_revision in changeset_revisions: