From edb51d876b763d0fd321c85d429575d465b754d1 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 15 Mar 2016 09:04:06 -0400 Subject: [PATCH 1/3] Rename tool_shed_get to url_get. --- lib/galaxy/util/__init__.py | 20 ++++++++++++++++ .../galaxy/api/tool_shed_repositories.py | 2 +- .../webapps/galaxy/controllers/admin.py | 2 +- .../galaxy/controllers/admin_toolshed.py | 14 +++++------ .../webapps/galaxy/controllers/workflow.py | 2 +- .../galaxy_install/dependency_display.py | 2 +- .../galaxy_install/install_manager.py | 4 ++-- .../repository_dependency_manager.py | 4 ++-- .../tool_dependencies/recipe/tag_handler.py | 4 ++-- .../galaxy_install/tool_migration_manager.py | 2 +- .../update_repository_manager.py | 3 ++- lib/tool_shed/util/common_util.py | 23 ++----------------- lib/tool_shed/util/readme_util.py | 4 ++-- lib/tool_shed/util/shed_util_common.py | 16 ++++++------- 14 files changed, 52 insertions(+), 50 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index be904c17afc..037480e28f4 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1342,6 +1342,26 @@ def parse_int(value, min_val=None, max_val=None, default=None, allow_none=False) raise +def url_get( app, base_url, toolshed=True, pathspec=None, params=None ): + """Make contact with the uri provided and return any contents.""" + # urllib2 auto-detects system proxies, when passed a Proxyhandler. + # Refer: https://docs.python.org/2/howto/urllib2.html#proxies + proxy = urllib2.ProxyHandler() + urlopener = urllib2.build_opener( proxy ) + urllib2.install_opener( urlopener ) + if toolshed: + registry = app.tool_shed_registry + password_mgr = registry.password_manager_for_url( base_url ) + if password_mgr is not None: + auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr ) + urlopener.add_handler( auth_handler ) + full_url = url_join( base_url, pathspec=pathspec, params=params ) + response = urlopener.open( full_url ) + content = response.read() + response.close() + return content + + def safe_relpath(path): """ Given what we expect to be a relative path, determine whether the path diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 215cd0b7b3c..7f8946512af 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -100,7 +100,7 @@ class ToolShedRepositoriesController( BaseAPIController ): params = dict(name=name, owner=owner) pathspec = ['api', 'repositories', 'get_ordered_installable_revisions'] try: - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec, params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec, params ) except Exception, e: message = "Error attempting to retrieve the latest installable revision from tool shed %s for repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( name ), str( owner ), str( e ) ) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 1f6109fb759..8fc36da84ce 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -781,7 +781,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP changeset_revision = elem.get( 'changeset_revision' ) params = dict( name=repository_name, owner='devteam', changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] - text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + text = url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) if text: tool_dependencies_dict = encoding_util.tool_shed_decode( text ) for dependency_key, requirements_dict in tool_dependencies_dict.items(): diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 4ca6110d570..d33314179b3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -413,7 +413,7 @@ class AdminToolshed( AdminGalaxy ): raise Exception( message ) params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) if len( raw_text ) > 2: encoded_text = json.loads( raw_text ) text = encoding_util.tool_shed_decode( encoded_text ) @@ -439,7 +439,7 @@ class AdminToolshed( AdminGalaxy ): owner=str( repository_owner ), changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_updated_repository_information' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) repo_information_dict = json.loads( raw_text ) return repo_information_dict @@ -529,7 +529,7 @@ class AdminToolshed( AdminGalaxy ): name=name, owner=owner ) pathspec = [ 'repository', 'get_latest_downloadable_changeset_revision' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) latest_downloadable_revision = json.loads( raw_text ) if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH: @@ -985,7 +985,7 @@ class AdminToolshed( AdminGalaxy ): try: params = dict( name=str( repository.name ), owner=str( repository.owner ) ) pathspec = [ 'repository', 'get_repository_id' ] - repository_ids = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + repository_ids = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) except Exception, e: # The Tool Shed cannot handle the get_repository_id request, so the code must be older than the # 04/2014 Galaxy release when it was introduced. It will be safest to error out and let the @@ -1005,7 +1005,7 @@ class AdminToolshed( AdminGalaxy ): # Get the information necessary to install each repository. params = dict( repository_ids=str( repository_ids ), changeset_revisions=str( changeset_revisions ) ) pathspec = [ 'repository', 'get_repository_information' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) repo_information_dict = json.loads( raw_text ) for encoded_repo_info_dict in repo_information_dict.get( 'repo_info_dicts', [] ): decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict ) @@ -1539,7 +1539,7 @@ class AdminToolshed( AdminGalaxy ): owner=tool_shed_repository.owner, changeset_revision=tool_shed_repository.installed_changeset_revision ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) tool_dependencies = metadata.get( 'tool_dependencies', None ) rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app ) @@ -1739,7 +1739,7 @@ class AdminToolshed( AdminGalaxy ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision ) pathspec = [ 'repository', 'get_tool_versions' ] - text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) if text: tool_version_dicts = json.loads( text ) tvm = tool_version_manager.ToolVersionManager( trans.app ) diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index c92a5fbf878..8a22cccd7aa 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -862,7 +862,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix workflow_name=encoding_util.tool_shed_encode( workflow_name ), open_for_url=True ) pathspec = [ 'workflow', 'import_workflow' ] - workflow_text = common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + workflow_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) import_button = True if import_button: workflow_data = None diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 3b92497f521..cfca0e39062 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -455,7 +455,7 @@ class DependencyDisplayer( object ): owner=str( repository.owner ), changeset_revision=str( repository.installed_changeset_revision ) ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) else: readme_files_dict = readme_util.build_readme_files_dict( self.app, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 8351a935777..9836084485c 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -460,7 +460,7 @@ class InstallRepositoryManager( object ): changeset_revision=str( changeset_revision ) ) pathspec = [ 'api', 'repositories', 'get_repository_revision_install_info' ] try: - raw_text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) except Exception, e: message = "Error attempting to retrieve installation information from tool shed " message += "%s for revision %s of repository %s owned by %s: %s" % \ @@ -1006,7 +1006,7 @@ def fetch_tool_versions( app, tool_shed_repository ): changeset_revision=str( tool_shed_repository.changeset_revision ) ) pathspec = [ 'repository', 'get_tool_versions' ] url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) if text: return json.loads( text ) else: diff --git a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py index b03417da16f..25f85e07a16 100644 --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -8,7 +8,7 @@ import os import urllib import urllib2 -from galaxy.util import asbool +from galaxy.util import asbool, url_get from tool_shed.galaxy_install.tools import tool_panel_manager @@ -291,7 +291,7 @@ class RepositoryDependencyInstallManager( object ): changeset_revision=str( repository.changeset_revision ) ) pathspec = [ 'repository', 'get_repository_dependencies' ] try: - raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = url_get( app, tool_shed_url, pathspec=pathspec, params=params ) except Exception, e: log.error("The URL\n%s\nraised the exception:\n%s\n", common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ), str( e ) ) return '' diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py index 2198ceb8512..40837103f3c 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -3,7 +3,7 @@ import os import tempfile from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY -from galaxy.util import listify +from galaxy.util import listify, url_get from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import shed_util_common as suc @@ -249,7 +249,7 @@ class Repository( RecipeTag, SyncDatabase ): owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies_config_contents' ] - text = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + text = url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) if text: # Write the contents to a temporary file on disk so it can be reloaded and parsed. fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-cttdc" ) diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index 64842c07d1f..a07b03035db 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -581,7 +581,7 @@ class ToolMigrationManager( object ): owner=self.repository_owner, changeset_revision=tool_shed_repository.installed_changeset_revision ) pathspec = [ 'repository', 'get_tool_versions' ] - text = common_util.tool_shed_get( self.app, self.tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( self.app, self.tool_shed_url, pathspec=pathspec, params=params ) if text: tool_version_dicts = json.loads( text ) tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) diff --git a/lib/tool_shed/galaxy_install/update_repository_manager.py b/lib/tool_shed/galaxy_install/update_repository_manager.py index 824f5237e10..24ab0d8c65f 100644 --- a/lib/tool_shed/galaxy_install/update_repository_manager.py +++ b/lib/tool_shed/galaxy_install/update_repository_manager.py @@ -9,6 +9,7 @@ from sqlalchemy import false import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import encoding_util +from galaxy import util log = logging.getLogger( __name__ ) @@ -36,7 +37,7 @@ class UpdateRepositoryManager( object ): changeset_revision=str( repository.installed_changeset_revision ) ) pathspec = [ 'repository', 'get_changeset_revision_and_ctx_rev' ] try: - encoded_update_dict = common_util.tool_shed_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + encoded_update_dict = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) if encoded_update_dict: update_dict = encoding_util.tool_shed_decode( encoded_update_dict ) includes_data_managers = update_dict.get( 'includes_data_managers', False ) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index a3d5f6561ce..ab66b9e0e86 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -160,7 +160,7 @@ def get_repository_dependencies( app, tool_shed_url, repository_name, repository params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_repository_dependencies' ] try: - raw_text = tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) tool_shed_accessible = True except Exception, e: tool_shed_accessible = False @@ -192,7 +192,7 @@ def get_tool_dependencies( app, tool_shed_url, repository_name, repository_owner params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] try: - text = tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) tool_shed_accessible = True except Exception, e: tool_shed_accessible = False @@ -341,25 +341,6 @@ def remove_protocol_from_tool_shed_url( tool_shed_url ): return util.remove_protocol_from_url( tool_shed_url ) -def tool_shed_get( app, base_url, pathspec=None, params=None ): - """Make contact with the tool shed via the uri provided.""" - registry = app.tool_shed_registry - # urllib2 auto-detects system proxies, when passed a Proxyhandler. - # Refer: https://docs.python.org/2/howto/urllib2.html#proxies - proxy = urllib2.ProxyHandler() - urlopener = urllib2.build_opener( proxy ) - urllib2.install_opener( urlopener ) - password_mgr = registry.password_manager_for_url( base_url ) - if password_mgr is not None: - auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr ) - urlopener.add_handler( auth_handler ) - full_url = url_join( base_url, pathspec=pathspec, params=params ) - response = urlopener.open( full_url ) - content = response.read() - response.close() - return content - - def url_join( base_url, pathspec=None, params=None ): """Return a valid URL produced by appending a base URL and a set of request parameters.""" url = base_url.rstrip( '/' ) diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index a68f01f1036..c6830da254f 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -7,7 +7,7 @@ from mako.template import Template import tool_shed.util.shed_util_common as suc from galaxy import web -from galaxy.util import rst_to_html, unicodify +from galaxy.util import rst_to_html, unicodify, url_get from tool_shed.util import basic_util, common_util, hg_util log = logging.getLogger( __name__ ) @@ -97,7 +97,7 @@ def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = url_get( app, tool_shed_url, pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) return readme_files_dict diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 9596585ec40..f6dad0e7c2f 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -301,7 +301,7 @@ def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_ctx_rev' ] - ctx_rev = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + ctx_rev = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return ctx_rev @@ -369,7 +369,7 @@ def get_tool_dependency_definition_metadata_from_tool_shed( app, tool_shed_url, tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner ) pathspec = [ 'repository', 'get_tool_dependency_definition_metadata' ] - metadata = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + metadata = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return metadata @@ -626,7 +626,7 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'next_installable_changeset_revision' ] - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) if text: repository = get_installed_repository( app=app, tool_shed=tool_shed, @@ -828,7 +828,7 @@ def get_repository_type_from_tool_shed( app, tool_shed_url, name, owner ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner ) pathspec = [ 'repository', 'get_repository_type' ] - repository_type = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + repository_type = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return repository_type @@ -914,7 +914,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ): params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision ) pathspec = [ 'repository', 'status_for_installed_repository' ] try: - encoded_tool_shed_status_dict = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + encoded_tool_shed_status_dict = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) tool_shed_status_dict = encoding_util.tool_shed_decode( encoded_tool_shed_status_dict ) return tool_shed_status_dict except HTTPError, e: @@ -926,7 +926,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ): params[ 'from_update_manager' ] = True try: # The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision. - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return dict( revision_update=text ) except Exception, e: # The required tool shed may be unavailable, so default the revision_update value to 'false'. @@ -1021,7 +1021,7 @@ def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, ow raise Exception( message ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'updated_changeset_revisions' ] - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) return text @@ -1227,7 +1227,7 @@ def repository_was_previously_installed( app, tool_shed_url, repository_name, re changeset_revision=changeset_revision, from_tip=str( from_tip ) ) pathspec = [ 'repository', 'previous_changeset_revisions' ] - text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) if text: changeset_revisions = util.listify( text ) for previous_changeset_revision in changeset_revisions: From 06b9737e0934823c807b02cbeaf590735437653d Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Wed, 23 Mar 2016 10:04:26 -0400 Subject: [PATCH 2/3] Cleanup and fix tests. --- lib/galaxy/util/__init__.py | 47 ++++++++++++++----- .../galaxy/api/tool_shed_repositories.py | 2 +- .../webapps/galaxy/controllers/admin.py | 8 ++-- .../galaxy/controllers/admin_toolshed.py | 30 ++++++------ .../webapps/galaxy/controllers/workflow.py | 2 +- .../tool_shed/controllers/repository.py | 8 ++-- lib/tool_shed/capsule/capsule_manager.py | 3 +- .../galaxy_install/dependency_display.py | 2 +- .../galaxy_install/install_manager.py | 6 +-- .../repository_dependency_manager.py | 8 ++-- .../tool_dependencies/recipe/tag_handler.py | 2 +- .../galaxy_install/tool_migration_manager.py | 2 +- .../update_repository_manager.py | 2 +- lib/tool_shed/scripts/api/common.py | 4 +- ...deprecate_repositories_without_metadata.py | 4 +- lib/tool_shed/tool_shed_registry.py | 11 +++-- lib/tool_shed/util/common_util.py | 24 +++------- lib/tool_shed/util/readme_util.py | 2 +- lib/tool_shed/util/shed_util_common.py | 18 +++---- 19 files changed, 100 insertions(+), 85 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 037480e28f4..3478801d726 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -30,6 +30,7 @@ from xml.etree import ElementInclude, ElementTree from six import binary_type, iteritems, PY3, string_types, text_type from six.moves import email_mime_text, xrange, zip from six.moves.urllib import parse as urlparse +from six.moves.urllib import request as urlrequest try: import docutils.core as docutils_core @@ -1342,20 +1343,40 @@ def parse_int(value, min_val=None, max_val=None, default=None, allow_none=False) raise -def url_get( app, base_url, toolshed=True, pathspec=None, params=None ): +def build_url( base_url, port=80, scheme='http', pathspec=None, params=None, doseq=False ): + if params is None: + params = dict() + if pathspec is None: + pathspec = [] + parsed_url = urlparse.urlparse( base_url ) + if scheme != 'http': + parsed_url.scheme = scheme + if port != 80: + url = '%s://%s:%d/%s' % ( parsed_url.scheme, parsed_url.netloc.rstrip( '/' ), int( port ), parsed_url.path ) + else: + url = '%s://%s/%s' % ( parsed_url.scheme, parsed_url.netloc.rstrip( '/' ), parsed_url.path.lstrip( '/' ) ) + if len( pathspec ) > 0: + url = '%s/%s' % ( url.rstrip( '/' ), '/'.join( pathspec ) ) + if parsed_url.query: + for query_parameter in parsed_url.query.split( '&' ): + key, value = query_parameter.split( '=' ) + params[ key ] = value + if params: + url += '?%s' % urlparse.urlencode( params, doseq=doseq ) + return url + + +def url_get( base_url, password_mgr=None, pathspec=None, params=None ): """Make contact with the uri provided and return any contents.""" - # urllib2 auto-detects system proxies, when passed a Proxyhandler. - # Refer: https://docs.python.org/2/howto/urllib2.html#proxies - proxy = urllib2.ProxyHandler() - urlopener = urllib2.build_opener( proxy ) - urllib2.install_opener( urlopener ) - if toolshed: - registry = app.tool_shed_registry - password_mgr = registry.password_manager_for_url( base_url ) - if password_mgr is not None: - auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr ) - urlopener.add_handler( auth_handler ) - full_url = url_join( base_url, pathspec=pathspec, params=params ) + # Uses system proxy settings if they exist. + proxy = urlrequest.ProxyHandler() + if password_mgr is not None: + auth = urlrequest.HTTPDigestAuthHandler( password_mgr ) + urlopener = urlrequest.build_opener( proxy, auth ) + else: + urlopener = urlrequest.build_opener( proxy ) + urlrequest.install_opener( urlopener ) + full_url = build_url( base_url, pathspec=pathspec, params=params ) response = urlopener.open( full_url ) content = response.read() response.close() diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 7f8946512af..2c01bd90cc1 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -100,7 +100,7 @@ class ToolShedRepositoriesController( BaseAPIController ): params = dict(name=name, owner=owner) pathspec = ['api', 'repositories', 'get_ordered_installable_revisions'] try: - raw_text = util.url_get( trans.app, tool_shed_url, pathspec, params ) + raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) except Exception, e: message = "Error attempting to retrieve the latest installable revision from tool shed %s for repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( name ), str( owner ), str( e ) ) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 8fc36da84ce..d3c1414c549 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -11,7 +11,7 @@ from galaxy.actions.admin import AdminActions from galaxy.exceptions import MessageException from galaxy.model import tool_shed_install as install_model from galaxy.model.util import pgcalc -from galaxy.util import nice_size, sanitize_text +from galaxy.util import nice_size, sanitize_text, url_get from galaxy.util.odict import odict from galaxy.web import url_for from galaxy.web.base.controller import BaseUIController, UsesQuotaMixin @@ -770,9 +770,9 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP tree = galaxy.util.parse_xml( tools_xml_file_path ) root = tree.getroot() tool_shed = root.get( 'name' ) - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed ) + shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed ) repo_name_dependency_tups = [] - if tool_shed_url: + if shed_url: for elem in root: if elem.tag == 'repository': tool_dependencies = [] @@ -781,7 +781,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP changeset_revision = elem.get( 'changeset_revision' ) params = dict( name=repository_name, owner='devteam', changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] - text = url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + text = url_get( shed_url, password_mgr=self.app.tool_shed_registry.url_auth( shed_url ), pathspec=pathspec, params=params ) if text: tool_dependencies_dict = encoding_util.tool_shed_decode( text ) for dependency_key, requirements_dict in tool_dependencies_dict.items(): diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index d33314179b3..624a26069df 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -189,7 +189,7 @@ class AdminToolshed( AdminGalaxy ): tool_shed_url = kwd.get( 'tool_shed_url', '' ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url ) params = dict( galaxy_url=web.url_for( '/', qualified=True ) ) - url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params ) + url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params ) return trans.response.send_redirect( url ) @web.expose @@ -212,7 +212,7 @@ class AdminToolshed( AdminGalaxy ): owner=str( repository.owner ), changeset_revision=str( repository.changeset_revision ) ) pathspec = [ 'repository', 'check_for_updates' ] - url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) + url = util.build_url( tool_shed_url, pathspec=pathspec, params=params ) return trans.response.send_redirect( url ) @web.expose @@ -369,7 +369,7 @@ class AdminToolshed( AdminGalaxy ): tool_shed_url = kwd.get( 'tool_shed_url', '' ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url ) params = dict( galaxy_url=web.url_for( '/', qualified=True ) ) - url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'find_tools' ], params=params ) + url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'find_tools' ], params=params ) return trans.response.send_redirect( url ) @web.expose @@ -378,7 +378,7 @@ class AdminToolshed( AdminGalaxy ): tool_shed_url = kwd.get( 'tool_shed_url', '' ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url ) params = dict( galaxy_url=web.url_for( '/', qualified=True ) ) - url = common_util.url_join( tool_shed_url, pathspec=[ 'repository', 'find_workflows' ], params=params ) + url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'find_workflows' ], params=params ) return trans.response.send_redirect( url ) @web.expose @@ -413,7 +413,7 @@ class AdminToolshed( AdminGalaxy ): raise Exception( message ) params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] - raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) if len( raw_text ) > 2: encoded_text = json.loads( raw_text ) text = encoding_util.tool_shed_decode( encoded_text ) @@ -439,7 +439,7 @@ class AdminToolshed( AdminGalaxy ): owner=str( repository_owner ), changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_updated_repository_information' ] - raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) repo_information_dict = json.loads( raw_text ) return repo_information_dict @@ -529,8 +529,8 @@ class AdminToolshed( AdminGalaxy ): name=name, owner=owner ) pathspec = [ 'repository', 'get_latest_downloadable_changeset_revision' ] - raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) - url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) + url = util.build_url( tool_shed_url, pathspec=pathspec, params=params ) latest_downloadable_revision = json.loads( raw_text ) if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH: message = 'Error retrieving the latest downloadable revision for this repository via the url %s.' % url @@ -541,7 +541,7 @@ class AdminToolshed( AdminGalaxy ): # appropriate repository revision if one exists. We need to create a temporary repo_info_tuple # with the following entries to handle this. # ( description, clone_url, changeset_revision, ctx_rev, owner, repository_dependencies, tool_dependencies ) - tmp_clone_url = common_util.url_join( tool_shed_url, pathspec=[ 'repos', owner, name ] ) + tmp_clone_url = util.build_url( tool_shed_url, pathspec=[ 'repos', owner, name ] ) tmp_repo_info_tuple = ( None, tmp_clone_url, latest_downloadable_revision, None, owner, None, None ) installed_repository, installed_changeset_revision = \ suc.repository_was_previously_installed( trans.app, tool_shed_url, name, tmp_repo_info_tuple, from_tip=False ) @@ -559,7 +559,7 @@ class AdminToolshed( AdminGalaxy ): changeset_revisions=str( latest_downloadable_revision ), galaxy_url=web.url_for( '/', qualified=True ) ) pathspec = [ 'repository', 'install_repositories_by_revision' ] - url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) + url = util.build_url( tool_shed_url, pathspec=pathspec, params=params ) return trans.response.send_redirect( url ) else: message = 'Cannot locate installed tool shed repository with encoded id %s.' % str( repository_id ) @@ -727,7 +727,7 @@ class AdminToolshed( AdminGalaxy ): changeset_revisions=installed_changeset_revision, galaxy_url=web.url_for( '/', qualified=True ) ) pathspec = [ 'repository', 'install_repositories_by_revision' ] - url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) + url = util.build_url( tool_shed_url, pathspec=pathspec, params=params ) return trans.response.send_redirect( url ) description = kwd.get( 'description', repository.description ) shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) @@ -985,7 +985,7 @@ class AdminToolshed( AdminGalaxy ): try: params = dict( name=str( repository.name ), owner=str( repository.owner ) ) pathspec = [ 'repository', 'get_repository_id' ] - repository_ids = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + repository_ids = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) except Exception, e: # The Tool Shed cannot handle the get_repository_id request, so the code must be older than the # 04/2014 Galaxy release when it was introduced. It will be safest to error out and let the @@ -1005,7 +1005,7 @@ class AdminToolshed( AdminGalaxy ): # Get the information necessary to install each repository. params = dict( repository_ids=str( repository_ids ), changeset_revisions=str( changeset_revisions ) ) pathspec = [ 'repository', 'get_repository_information' ] - raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) repo_information_dict = json.loads( raw_text ) for encoded_repo_info_dict in repo_information_dict.get( 'repo_info_dicts', [] ): decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict ) @@ -1539,7 +1539,7 @@ class AdminToolshed( AdminGalaxy ): owner=tool_shed_repository.owner, changeset_revision=tool_shed_repository.installed_changeset_revision ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) tool_dependencies = metadata.get( 'tool_dependencies', None ) rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app ) @@ -1739,7 +1739,7 @@ class AdminToolshed( AdminGalaxy ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision ) pathspec = [ 'repository', 'get_tool_versions' ] - text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) if text: tool_version_dicts = json.loads( text ) tvm = tool_version_manager.ToolVersionManager( trans.app ) diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index 8a22cccd7aa..ebd6ebfab70 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -862,7 +862,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix workflow_name=encoding_util.tool_shed_encode( workflow_name ), open_for_url=True ) pathspec = [ 'workflow', 'import_workflow' ] - workflow_text = util.url_get( trans.app, tool_shed_url, pathspec=pathspec, params=params ) + workflow_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) import_button = True if import_button: workflow_data = None diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index df8530c33f0..646d470f823 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -809,7 +809,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return update params['latest_changeset_revision'] = str( latest_changeset_revision ) params['latest_ctx_rev'] = str( update_to_ctx.rev() ) - url = common_util.url_join( galaxy_url, pathspec=pathspec, params=params ) + url = util.build_url( galaxy_url, pathspec=pathspec, params=params ) return trans.response.send_redirect( url ) @web.expose @@ -1065,7 +1065,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): trans.sa_session.flush() tool_shed_url = web.url_for( '/', qualified=True ) pathspec = [ 'repos', str( repository.user.username ), str( repository.name ), 'archive', file_type_str ] - download_url = common_util.url_join( tool_shed_url, pathspec=pathspec ) + download_url = util.build_url( tool_shed_url, pathspec=pathspec ) return trans.response.send_redirect( download_url ) @web.expose @@ -1534,7 +1534,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): time_tested = repository_metadata.time_last_tested.strftime( '%a, %d %b %Y %H:%M:%S UT' ) # Generate a citable URL for this repository with owner and changeset revision. pathspec = [ 'view', str( user.username ), str( repository.name ), str( repository_metadata.changeset_revision ) ] - repository_citable_url = common_util.url_join( tool_shed_url, pathspec=pathspec ) + repository_citable_url = util.build_url( tool_shed_url, pathspec=pathspec ) passed_tests = len( tool_test_results.get( 'passed_tests', [] ) ) failed_tests = len( tool_test_results.get( 'failed_tests', [] ) ) missing_test_components = len( tool_test_results.get( 'missing_test_components', [] ) ) @@ -1982,7 +1982,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): repository_ids=','.join( util.listify( repository_ids ) ), changeset_revisions=','.join( util.listify( changeset_revisions ) ) ) pathspec = [ 'admin_toolshed', 'prepare_for_install' ] - url = common_util.url_join( galaxy_url, pathspec=pathspec, params=params ) + url = util.build_url( galaxy_url, pathspec=pathspec, params=params ) return trans.response.send_redirect( url ) else: message = 'Repository installation is not possible due to an invalid Galaxy URL: %s. ' % galaxy_url diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 7dd879bf6eb..5fcd84c5b18 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -15,6 +15,7 @@ from galaxy import web from galaxy.util import asbool from galaxy.util import CHUNK_SIZE from galaxy.util import safe_relpath +from galaxy.util import build_url from galaxy.util.odict import odict from tool_shed.dependencies.repository.relation_builder import RelationBuilder from tool_shed.dependencies import attribute_handlers @@ -123,7 +124,7 @@ class ExportRepositoryManager( object ): params = dict( encoded_repositories_archive_name=encoded_repositories_archive_name ) pathspec = [ 'repository', 'export_via_api' ] tool_shed_url = web.url_for( '/', qualified=True ) - download_url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) + download_url = build_url( tool_shed_url, pathspec=pathspec, params=params ) return dict( download_url=download_url, error_messages=error_messages ) return repositories_archive, error_messages diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index cfca0e39062..b5600bf3c66 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -455,7 +455,7 @@ class DependencyDisplayer( object ): owner=str( repository.owner ), changeset_revision=str( repository.installed_changeset_revision ) ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) else: readme_files_dict = readme_util.build_readme_files_dict( self.app, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 9836084485c..a03d8e72979 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -460,7 +460,7 @@ class InstallRepositoryManager( object ): changeset_revision=str( changeset_revision ) ) pathspec = [ 'api', 'repositories', 'get_repository_revision_install_info' ] try: - raw_text = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) except Exception, e: message = "Error attempting to retrieve installation information from tool shed " message += "%s for revision %s of repository %s owned by %s: %s" % \ @@ -1005,8 +1005,8 @@ def fetch_tool_versions( app, tool_shed_repository ): owner=str( tool_shed_repository.owner ), changeset_revision=str( tool_shed_repository.changeset_revision ) ) pathspec = [ 'repository', 'get_tool_versions' ] - url = common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ) - text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + url = util.build_url( tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) if text: return json.loads( text ) else: diff --git a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py index 25f85e07a16..a69c0357aec 100644 --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -8,7 +8,7 @@ import os import urllib import urllib2 -from galaxy.util import asbool, url_get +from galaxy.util import asbool, url_get, build_url from tool_shed.galaxy_install.tools import tool_panel_manager @@ -291,9 +291,9 @@ class RepositoryDependencyInstallManager( object ): changeset_revision=str( repository.changeset_revision ) ) pathspec = [ 'repository', 'get_repository_dependencies' ] try: - raw_text = url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) except Exception, e: - log.error("The URL\n%s\nraised the exception:\n%s\n", common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ), str( e ) ) + log.error("The URL\n%s\nraised the exception:\n%s\n", build_url( tool_shed_url, pathspec=pathspec, params=params ), str( e ) ) return '' if len( raw_text ) > 2: encoded_text = json.loads( raw_text ) @@ -378,7 +378,7 @@ class RepositoryDependencyInstallManager( object ): # Handle secure / insecure Tool Shed URL protocol changes and port changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url ) pathspec = [ 'repository', 'get_required_repo_info_dict' ] - url = common_util.url_join( tool_shed_url, pathspec=pathspec ) + url = build_url( tool_shed_url, pathspec=pathspec ) # Fix for handling 307 redirect not being handled nicely by urllib2.urlopen when the urllib2.Request has data provided url = urllib2.urlopen( urllib2.Request( url ) ).geturl() request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) ) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py index 40837103f3c..d505325a8be 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -249,7 +249,7 @@ class Repository( RecipeTag, SyncDatabase ): owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies_config_contents' ] - text = url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + text = url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) if text: # Write the contents to a temporary file on disk so it can be reloaded and parsed. fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-cttdc" ) diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index a07b03035db..2596e03db86 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -581,7 +581,7 @@ class ToolMigrationManager( object ): owner=self.repository_owner, changeset_revision=tool_shed_repository.installed_changeset_revision ) pathspec = [ 'repository', 'get_tool_versions' ] - text = util.url_get( self.app, self.tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( self.tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) if text: tool_version_dicts = json.loads( text ) tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) diff --git a/lib/tool_shed/galaxy_install/update_repository_manager.py b/lib/tool_shed/galaxy_install/update_repository_manager.py index 24ab0d8c65f..aa8e60e9b61 100644 --- a/lib/tool_shed/galaxy_install/update_repository_manager.py +++ b/lib/tool_shed/galaxy_install/update_repository_manager.py @@ -37,7 +37,7 @@ class UpdateRepositoryManager( object ): changeset_revision=str( repository.installed_changeset_revision ) ) pathspec = [ 'repository', 'get_changeset_revision_and_ctx_rev' ] try: - encoded_update_dict = util.url_get( self.app, tool_shed_url, pathspec=pathspec, params=params ) + encoded_update_dict = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) if encoded_update_dict: update_dict = encoding_util.tool_shed_decode( encoded_update_dict ) includes_data_managers = update_dict.get( 'includes_data_managers', False ) diff --git a/lib/tool_shed/scripts/api/common.py b/lib/tool_shed/scripts/api/common.py index 02e2b3b1a6a..918a8f59244 100644 --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -7,7 +7,7 @@ import urllib2 sys.path.insert(1, os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir, os.pardir ) ) from tool_shed.util import common_util, hg_util - +from galaxy import util class HTTPRedirectWithDataHandler( urllib2.HTTPRedirectHandler ): @@ -131,7 +131,7 @@ def get_api_url( base, parts=[], params=None ): parts.insert( 0, 'api' ) elif 'api' not in parts: parts.insert( 0, 'api' ) - url = common_util.url_join( base, pathspec=parts, params=params ) + url = util.build_url( base, pathspec=parts, params=params ) return url diff --git a/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py b/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py index ed86ce60be8..3f090f2edba 100644 --- a/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py +++ b/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py @@ -19,7 +19,7 @@ from sqlalchemy import and_, distinct, false, not_ import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping from galaxy.util import send_mail as galaxy_send_mail -from tool_shed.util.common_util import url_join +from galaxy.util import build_url log = logging.getLogger() log.setLevel( 10 ) @@ -28,7 +28,7 @@ assert sys.version_info[:2] >= ( 2, 4 ) def build_citable_url( host, repository ): - return url_join( host, pathspec=[ 'view', repository.user.username, repository.name ] ) + return build_url( host, pathspec=[ 'view', repository.user.username, repository.name ] ) def main(): diff --git a/lib/tool_shed/tool_shed_registry.py b/lib/tool_shed/tool_shed_registry.py index 5b26bf62395..6014ee5b414 100644 --- a/lib/tool_shed/tool_shed_registry.py +++ b/lib/tool_shed/tool_shed_registry.py @@ -1,8 +1,8 @@ import logging -import urllib2 from galaxy.util.odict import odict from tool_shed.util import common_util, xml_util +from six.moves.urllib import request as urlrequest log = logging.getLogger( __name__ ) @@ -31,7 +31,7 @@ class Registry( object ): self.tool_sheds_auth[ name ] = None log.debug( 'Loaded reference to tool shed: %s' % name ) if name and url and username and password: - pass_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + pass_mgr = urlrequest.HTTPPasswordMgrWithDefaultRealm() pass_mgr.add_password( None, url, username, password ) self.tool_sheds_auth[ name ] = pass_mgr except Exception, e: @@ -40,7 +40,7 @@ class Registry( object ): def password_manager_for_url( self, url ): """ If the tool shed is using external auth, the client to the tool shed must authenticate to that - as well. This provides access to the urllib2.HTTPPasswordMgrWithdefaultRealm() object for the + as well. This provides access to the six.moves.urllib.request.HTTPPasswordMgrWithdefaultRealm() object for the url passed in. Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at @@ -53,3 +53,8 @@ class Registry( object ): return self.tool_sheds_auth[ shed_name ] log.debug( "Invalid url '%s' received by tool shed registry's password_manager_for_url method." % str( url ) ) return None + + def url_auth( self, url ): + password_manager = self.password_manager_for_url( url ) + if password_manager is not None: + return urlrequest.HTTPBasicAuthHandler( password_manager ) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index ab66b9e0e86..974efd3072f 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -114,7 +114,7 @@ def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_d def generate_clone_url_for_installed_repository( app, repository ): """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" tool_shed_url = get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) ) - return url_join( tool_shed_url, pathspec=[ 'repos', str( repository.owner ), str( repository.name ) ] ) + return util.build_url( tool_shed_url, pathspec=[ 'repos', str( repository.owner ), str( repository.name ) ] ) def generate_clone_url_for_repository_in_tool_shed( user, repository ): @@ -135,7 +135,7 @@ def generate_clone_url_from_repo_info_tup( app, repo_info_tup ): parse_repository_dependency_tuple( repo_info_tup ) tool_shed_url = get_tool_shed_url_from_tool_shed_registry( app, toolshed ) # Don't include the changeset_revision in clone urls. - return url_join( tool_shed_url, pathspec=[ 'repos', owner, name ] ) + return util.build_url( tool_shed_url, pathspec=[ 'repos', owner, name ] ) def get_non_shed_tool_panel_configs( app ): @@ -160,11 +160,11 @@ def get_repository_dependencies( app, tool_shed_url, repository_name, repository params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_repository_dependencies' ] try: - raw_text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) tool_shed_accessible = True except Exception, e: tool_shed_accessible = False - log.warn( "The URL\n%s\nraised the exception:\n%s\n", url_join( tool_shed_url, pathspec=pathspec, params=params ), e ) + log.warn( "The URL\n%s\nraised the exception:\n%s\n", util.build_url( tool_shed_url, pathspec=pathspec, params=params ), e ) if tool_shed_accessible: if len( raw_text ) > 2: encoded_text = json.loads( raw_text ) @@ -192,11 +192,11 @@ def get_tool_dependencies( app, tool_shed_url, repository_name, repository_owner params = dict( name=repository_name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_tool_dependencies' ] try: - text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) tool_shed_accessible = True except Exception, e: tool_shed_accessible = False - log.warn( "The URL\n%s\nraised the exception:\n%s\n", url_join( tool_shed_url, pathspec=pathspec, params=params ), e ) + log.warn( "The URL\n%s\nraised the exception:\n%s\n", util.build_url( tool_shed_url, pathspec=pathspec, params=params ), e ) if tool_shed_accessible: if text: tool_dependencies_dict = encoding_util.tool_shed_decode( text ) @@ -339,15 +339,3 @@ def remove_protocol_and_user_from_clone_url( repository_clone_url ): def remove_protocol_from_tool_shed_url( tool_shed_url ): """Return a partial Tool Shed URL, eliminating the protocol if it exists.""" return util.remove_protocol_from_url( tool_shed_url ) - - -def url_join( base_url, pathspec=None, params=None ): - """Return a valid URL produced by appending a base URL and a set of request parameters.""" - url = base_url.rstrip( '/' ) - if pathspec is not None: - if not isinstance( pathspec, string_types ): - pathspec = '/'.join( pathspec ) - url = '%s/%s' % ( url, pathspec ) - if params is not None: - url = '%s?%s' % ( url, urllib.urlencode( params ) ) - return url diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index c6830da254f..82b919db440 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -97,7 +97,7 @@ def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_readme_files' ] - raw_text = url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + raw_text = url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) return readme_files_dict diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index f6dad0e7c2f..5c0fefa7e63 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -254,7 +254,7 @@ def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_r tool_shed_url = items[ 0 ] repo_path = items[ 1 ] tool_shed_url = common_util.remove_port_from_tool_shed_url( tool_shed_url ) - return common_util.url_join( tool_shed_url, pathspec=[ 'repos', repo_path, changeset_revision ] ) + return '/'.join( [ tool_shed_url, 'repos', repo_path, changeset_revision ] ) def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): @@ -301,7 +301,7 @@ def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_ctx_rev' ] - ctx_rev = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + ctx_rev = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) return ctx_rev @@ -369,7 +369,7 @@ def get_tool_dependency_definition_metadata_from_tool_shed( app, tool_shed_url, tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner ) pathspec = [ 'repository', 'get_tool_dependency_definition_metadata' ] - metadata = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + metadata = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) return metadata @@ -626,7 +626,7 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'next_installable_changeset_revision' ] - text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) if text: repository = get_installed_repository( app=app, tool_shed=tool_shed, @@ -828,7 +828,7 @@ def get_repository_type_from_tool_shed( app, tool_shed_url, name, owner ): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=owner ) pathspec = [ 'repository', 'get_repository_type' ] - repository_type = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + repository_type = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) return repository_type @@ -914,7 +914,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ): params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision ) pathspec = [ 'repository', 'status_for_installed_repository' ] try: - encoded_tool_shed_status_dict = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + encoded_tool_shed_status_dict = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) tool_shed_status_dict = encoding_util.tool_shed_decode( encoded_tool_shed_status_dict ) return tool_shed_status_dict except HTTPError, e: @@ -926,7 +926,7 @@ def get_tool_shed_status_for_installed_repository( app, repository ): params[ 'from_update_manager' ] = True try: # The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision. - text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) return dict( revision_update=text ) except Exception, e: # The required tool shed may be unavailable, so default the revision_update value to 'false'. @@ -1021,7 +1021,7 @@ def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, ow raise Exception( message ) params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'updated_changeset_revisions' ] - text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) return text @@ -1227,7 +1227,7 @@ def repository_was_previously_installed( app, tool_shed_url, repository_name, re changeset_revision=changeset_revision, from_tip=str( from_tip ) ) pathspec = [ 'repository', 'previous_changeset_revisions' ] - text = util.url_get( app, tool_shed_url, pathspec=pathspec, params=params ) + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) if text: changeset_revisions = util.listify( text ) for previous_changeset_revision in changeset_revisions: From 7e14f5db6236a5ab1b9df988fe88cd34eaf8ba78 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Wed, 23 Mar 2016 11:40:20 -0400 Subject: [PATCH 3/3] Address travis' complaints. --- lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py | 1 - lib/galaxy/webapps/galaxy/controllers/workflow.py | 1 - lib/tool_shed/galaxy_install/tool_migration_manager.py | 2 +- lib/tool_shed/scripts/api/common.py | 3 ++- lib/tool_shed/util/common_util.py | 3 --- 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 2c01bd90cc1..d771b998f58 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -13,7 +13,6 @@ from galaxy.web.base.controller import BaseAPIController from tool_shed.galaxy_install.install_manager import InstallRepositoryManager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager -from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import workflow_util diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index ebd6ebfab70..7fcfa825131 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -10,7 +10,6 @@ from sqlalchemy import and_ from sqlalchemy.sql import expression from markupsafe import escape -from tool_shed.util import common_util from tool_shed.util import encoding_util from galaxy import model diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index 2596e03db86..104a0d84547 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -581,7 +581,7 @@ class ToolMigrationManager( object ): owner=self.repository_owner, changeset_revision=tool_shed_repository.installed_changeset_revision ) pathspec = [ 'repository', 'get_tool_versions' ] - text = util.url_get( self.tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) + text = util.url_get( self.tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( self.tool_shed_url ), pathspec=pathspec, params=params ) if text: tool_version_dicts = json.loads( text ) tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) diff --git a/lib/tool_shed/scripts/api/common.py b/lib/tool_shed/scripts/api/common.py index 918a8f59244..88f7fcf2e94 100644 --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -6,9 +6,10 @@ import urllib2 sys.path.insert(1, os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir, os.pardir ) ) -from tool_shed.util import common_util, hg_util +from tool_shed.util import hg_util from galaxy import util + class HTTPRedirectWithDataHandler( urllib2.HTTPRedirectHandler ): def __init__( self, method ): diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index 974efd3072f..2db00a001ea 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -1,9 +1,6 @@ import json import logging import os -import urllib -import urllib2 -from six import string_types from galaxy import util from galaxy.util.odict import odict