diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index be904c17afc..037480e28f4 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1342,6 +1342,26 @@ def parse_int(value, min_val=None, max_val=None, default=None, allow_none=False) raise +def url_get( app, base_url, toolshed=True, pathspec=None, params=None ): + """Make contact with the uri provided and return any contents.""" + # urllib2 auto-detects system proxies, when passed a Proxyhandler. + # Refer: https://docs.python.org/2/howto/urllib2.html#proxies + proxy = urllib2.ProxyHandler() + urlopener = urllib2.build_opener( proxy ) + urllib2.install_opener( urlopener ) + if toolshed: + registry = app.tool_shed_registry + password_mgr = registry.password_manager_for_url( base_url ) + if password_mgr is not None: + auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr ) + urlopener.add_handler( auth_handler ) + full_url = url_join( base_url, pathspec=pathspec, params=params ) + response = urlopener.open( full_url ) + content = response.read() + response.close() + return content + + def safe_relpath(path): """ Given what we expect to be a relative path, determine whether the path diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 215cd0b7b3c..7f8946512af 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -100,7 +100,7 @@ class ToolShedRepositoriesController( BaseAPIController ): params = dict(name=name, owner=owner) pathspec = ['api', 'repositories', 'get_ordered_installable_revisions'] try: - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec, params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec, params ) except Exception, e: message = "Error attempting to retrieve the latest installable revision from tool shed %s for repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( name ), str( owner ), str( e ) ) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 1f6109fb759..8fc36da84ce 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -781,7 +781,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP changeset_revision = elem.get( 'changeset_revision' ) params = dict( name=repository_name, owner='devteam', changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] - text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + text = url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) if text: tool_dependencies_dict = encoding_util.tool_shed_decode( text ) for dependency_key, requirements_dict in tool_dependencies_dict.items(): diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 4ca6110d570..d33314179b3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -413,7 +413,7 @@ class AdminToolshed( AdminGalaxy ): raise Exception( message ) params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) if len( raw_text ) > 2: encoded_text = json.loads( raw_text ) text = encoding_util.tool_shed_decode( encoded_text ) @@ -439,7 +439,7 @@ class AdminToolshed( AdminGalaxy ): owner=str( repository_owner ), changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_updated_repository_information' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) repo_information_dict = json.loads( raw_text ) return repo_information_dict @@ -529,7 +529,7 @@ class AdminToolshed( AdminGalaxy ): name=name, owner=owner ) pathspec = [ 'repository', 'get_latest_downloadable_changeset_revision' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) latest_downloadable_revision = json.loads( raw_text ) if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH: @@ -985,7 +985,7 @@ class AdminToolshed( AdminGalaxy ): try: params = dict( name=str( repository.name ), owner=str( repository.owner ) ) pathspec = [ 'repository', 'get_repository_id' ] - repository_ids = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + repository_ids = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) except Exception, e: # The Tool Shed cannot handle the get_repository_id request, so the code must be older than the # 04/2014 Galaxy release when it was introduced. It will be safest to error out and let the @@ -1005,7 +1005,7 @@ class AdminToolshed( AdminGalaxy ): # Get the information necessary to install each repository. params = dict( repository_ids=str( repository_ids ), changeset_revisions=str( changeset_revisions ) ) pathspec = [ 'repository', 'get_repository_information' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) repo_information_dict = json.loads( raw_text ) for encoded_repo_info_dict in repo_information_dict.get( 'repo_info_dicts', [] ): decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict ) @@ -1539,7 +1539,7 @@ class AdminToolshed( AdminGalaxy ): owner=tool_shed_repository.owner, changeset_revision=tool_shed_repository.installed_changeset_revision ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) tool_dependencies = metadata.get( 'tool_dependencies', None ) rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app ) @@ -1739,7 +1739,7 @@ class AdminToolshed( AdminGalaxy ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision ) pathspec = [ 'repository', 'get_tool_versions' ] - text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) if text: tool_version_dicts = json.loads( text ) tvm = tool_version_manager.ToolVersionManager( trans.app ) diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index c92a5fbf878..8a22cccd7aa 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -862,7 +862,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix workflow_name=encoding_util.tool_shed_encode( workflow_name ), open_for_url=True ) pathspec = [ 'workflow', 'import_workflow' ] - workflow_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + workflow_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) import_button = True if import_button: workflow_data = None diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 3b92497f521..cfca0e39062 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -455,7 +455,7 @@ class DependencyDisplayer( object ): owner=str( repository.owner ), changeset_revision=str( repository.installed_changeset_revision ) ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) else: readme_files_dict = readme_util.build_readme_files_dict( self.app, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 8351a935777..9836084485c 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -460,7 +460,7 @@ class InstallRepositoryManager( object ): changeset_revision=str( changeset_revision ) ) pathspec = [ 'api', 'repositories', 'get_repository_revision_install_info' ] try: - raw_text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) except Exception, e: message = "Error attempting to retrieve installation information from tool shed " message += "%s for revision %s of repository %s owned by %s: %s" % \ @@ -1006,7 +1006,7 @@ def fetch_tool_versions( app, tool_shed_repository ): changeset_revision=str( tool_shed_repository.changeset_revision ) ) pathspec = [ 'repository', 'get_tool_versions' ] url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) if text: return json.loads( text ) else: diff --git a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py index b03417da16f..25f85e07a16 100644 --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -8,7 +8,7 @@ import os import urllib import urllib2 -from galaxy.util import asbool +from galaxy.util import asbool, url_get from tool_shed.galaxy_install.tools import tool_panel_manager @@ -291,7 +291,7 @@ class RepositoryDependencyInstallManager( object ): changeset_revision=str( repository.changeset_revision ) ) pathspec = [ 'repository', 'get_repository_dependencies' ] try: - raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = url_get( app, tool_shed_url, pathspec=pathspec, params=params ) except Exception, e: log.error("The URL\n%s\nraised the exception:\n%s\n", common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ), str( e ) ) return '' diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py index 2198ceb8512..40837103f3c 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -3,7 +3,7 @@ import os import tempfile from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY -from galaxy.util import listify +from galaxy.util import listify, url_get from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import shed_util_common as suc @@ -249,7 +249,7 @@ class Repository( RecipeTag, SyncDatabase ): owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies_config_contents' ] - text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + text = url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) if text: # Write the contents to a temporary file on disk so it can be reloaded and parsed. fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-cttdc" ) diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index 64842c07d1f..a07b03035db 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -581,7 +581,7 @@ class ToolMigrationManager( object ): owner=self.repository_owner, changeset_revision=tool_shed_repository.installed_changeset_revision ) pathspec = [ 'repository', 'get_tool_versions' ] - text = common_util.tool_shed_get( self.app, self.tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( self.app, self.tool_shed_url, pathspec=pathspec, params=params ) if text: tool_version_dicts = json.loads( text ) tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) diff --git a/lib/tool_shed/galaxy_install/update_repository_manager.py b/lib/tool_shed/galaxy_install/update_repository_manager.py index 824f5237e10..24ab0d8c65f 100644 --- a/lib/tool_shed/galaxy_install/update_repository_manager.py +++ b/lib/tool_shed/galaxy_install/update_repository_manager.py @@ -9,6 +9,7 @@ from sqlalchemy import false import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import encoding_util +from galaxy import util log = logging.getLogger( __name__ ) @@ -36,7 +37,7 @@ class UpdateRepositoryManager( object ): changeset_revision=str( repository.installed_changeset_revision ) ) pathspec = [ 'repository', 'get_changeset_revision_and_ctx_rev' ] try: - encoded_update_dict = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + encoded_update_dict = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) if encoded_update_dict: update_dict = encoding_util.tool_shed_decode( encoded_update_dict ) includes_data_managers = update_dict.get( 'includes_data_managers', False ) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index a3d5f6561ce..ab66b9e0e86 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -160,7 +160,7 @@ def get_repository_dependencies( app, tool_shed_url, repository_name, repository params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_repository_dependencies' ] try: - raw_text = tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) tool_shed_accessible = True except Exception, e: tool_shed_accessible = False @@ -192,7 +192,7 @@ def get_tool_dependencies( app, tool_shed_url, repository_name, repository_owner params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] try: - text = tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) tool_shed_accessible = True except Exception, e: tool_shed_accessible = False @@ -341,25 +341,6 @@ def remove_protocol_from_tool_shed_url( tool_shed_url ): return util.remove_protocol_from_url( tool_shed_url ) -def tool_shed_get( app, base_url, pathspec=None, params=None ): - """Make contact with the tool shed via the uri provided.""" - registry = app.tool_shed_registry - # urllib2 auto-detects system proxies, when passed a Proxyhandler. - # Refer: https://docs.python.org/2/howto/urllib2.html#proxies - proxy = urllib2.ProxyHandler() - urlopener = urllib2.build_opener( proxy ) - urllib2.install_opener( urlopener ) - password_mgr = registry.password_manager_for_url( base_url ) - if password_mgr is not None: - auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr ) - urlopener.add_handler( auth_handler ) - full_url = url_join( base_url, pathspec=pathspec, params=params ) - response = urlopener.open( full_url ) - content = response.read() - response.close() - return content - - def url_join( base_url, pathspec=None, params=None ): """Return a valid URL produced by appending a base URL and a set of request parameters.""" url = base_url.rstrip( '/' ) diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index a68f01f1036..c6830da254f 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -7,7 +7,7 @@ from mako.template import Template import tool_shed.util.shed_util_common as suc from galaxy import web -from galaxy.util import rst_to_html, unicodify +from galaxy.util import rst_to_html, unicodify, url_get from tool_shed.util import basic_util, common_util, hg_util log = logging.getLogger( __name__ ) @@ -97,7 +97,7 @@ def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = url_get( app, tool_shed_url, pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) return readme_files_dict diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 9596585ec40..f6dad0e7c2f 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -301,7 +301,7 @@ def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_ctx_rev' ] - ctx_rev = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + ctx_rev = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return ctx_rev @@ -369,7 +369,7 @@ def get_tool_dependency_definition_metadata_from_tool_shed( app, tool_shed_url, tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner ) pathspec = [ 'repository', 'get_tool_dependency_definition_metadata' ] - metadata = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + metadata = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return metadata @@ -626,7 +626,7 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'next_installable_changeset_revision' ] - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) if text: repository = get_installed_repository( app=app, tool_shed=tool_shed, @@ -828,7 +828,7 @@ def get_repository_type_from_tool_shed( app, tool_shed_url, name, owner ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner ) pathspec = [ 'repository', 'get_repository_type' ] - repository_type = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + repository_type = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return repository_type @@ -914,7 +914,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ): params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision ) pathspec = [ 'repository', 'status_for_installed_repository' ] try: - encoded_tool_shed_status_dict = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + encoded_tool_shed_status_dict = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) tool_shed_status_dict = encoding_util.tool_shed_decode( encoded_tool_shed_status_dict ) return tool_shed_status_dict except HTTPError, e: @@ -926,7 +926,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ): params[ 'from_update_manager' ] = True try: # The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision. - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return dict( revision_update=text ) except Exception, e: # The required tool shed may be unavailable, so default the revision_update value to 'false'. @@ -1021,7 +1021,7 @@ def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, ow raise Exception( message ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'updated_changeset_revisions' ] - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return text @@ -1227,7 +1227,7 @@ def repository_was_previously_installed( app, tool_shed_url, repository_name, re changeset_revision=changeset_revision, from_tip=str( from_tip ) ) pathspec = [ 'repository', 'previous_changeset_revisions' ] - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) if text: changeset_revisions = util.listify( text ) for previous_changeset_revision in changeset_revisions: