opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python3 support and import order fixes for lib/tool_shed/

Browse Source
This commit is contained in:
Nicola Soranzo
2016-08-04 13:41:36 +01:00
parent 7efa1e0e4d
commit de3c50a5ef
50 changed files with 153 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/flake8_lint_include_list.txt
+1 -39
View File
@@ -384,45 +384,7 @@ lib/galaxy/workflow/steps.py
lib/galaxy/work/__init__.py
lib/mimeparse.py
lib/psyco_full.py
lib/tool_shed/capsule/__init__.py
lib/tool_shed/dependencies/__init__.py
lib/tool_shed/dependencies/repository/__init__.py
lib/tool_shed/dependencies/tool/
lib/tool_shed/galaxy_install/datatypes/__init__.py
lib/tool_shed/galaxy_install/grids/__init__.py
lib/tool_shed/galaxy_install/__init__.py
lib/tool_shed/galaxy_install/metadata/__init__.py
lib/tool_shed/galaxy_install/migrate/__init__.py
lib/tool_shed/galaxy_install/migrate/versions/
lib/tool_shed/galaxy_install/repository_dependencies/__init__.py
lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py
lib/tool_shed/galaxy_install/tool_dependencies/__init__.py
lib/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py
lib/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py
lib/tool_shed/galaxy_install/tool_dependencies/recipe/__init__.py
lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py
lib/tool_shed/galaxy_install/tools/
lib/tool_shed/galaxy_install/utility_containers/
lib/tool_shed/grids/__init__.py
lib/tool_shed/grids/repository_grid_filter_manager.py
lib/tool_shed/__init__.py
lib/tool_shed/managers/groups.py
lib/tool_shed/managers/__init__.py
lib/tool_shed/metadata/__init__.py
lib/tool_shed/repository_types/__init__.py
lib/tool_shed/repository_types/metadata.py
lib/tool_shed/repository_types/unrestricted.py
lib/tool_shed/repository_types/util.py
lib/tool_shed/tools/data_table_manager.py
lib/tool_shed/tools/__init__.py
lib/tool_shed/tools/tool_version_manager.py
lib/tool_shed/util/basic_util.py
lib/tool_shed/util/container_util.py
lib/tool_shed/util/encoding_util.py
lib/tool_shed/util/__init__.py
lib/tool_shed/util/search_util.py
lib/tool_shed/util/web_util.py
lib/tool_shed/util/xml_util.py
lib/tool_shed/
scripts/api/common.py
scripts/api/copy_hda_to_library_folder.py
scripts/api/create.py
.ci/py3_sources.txt
+1 -9
View File
@@ -50,15 +50,7 @@ lib/galaxy_ext/
lib/galaxy_utils/
lib/log_tempfile.py
lib/psyco_full.py
lib/tool_shed/capsule/
lib/tool_shed/dependencies/
lib/tool_shed/grids/
lib/tool_shed/managers/
lib/tool_shed/metadata/
lib/tool_shed/repository_types/
lib/tool_shed/tools/
lib/tool_shed/util/
lib/tool_shed/utility_containers/
lib/tool_shed/
scripts/api/
scripts/auth/
scripts/bootstrap_history.py
lib/tool_shed/capsule/capsule_manager.py
+7 -18
View File
@@ -5,32 +5,21 @@ import shutil
import tarfile
import tempfile
import threading
import urllib
from time import gmtime
from time import strftime
from time import gmtime, strftime
from six.moves.urllib.request import urlopen
from sqlalchemy import and_, false
import tool_shed.repository_types.util as rt_util
from galaxy import web
from galaxy.util import asbool
from galaxy.util import CHUNK_SIZE
from galaxy.util import safe_relpath
from galaxy.util import build_url
from galaxy.util import asbool, build_url, CHUNK_SIZE, safe_relpath
from galaxy.util.odict import odict
from tool_shed.dependencies.repository.relation_builder import RelationBuilder
from tool_shed.dependencies import attribute_handlers
from tool_shed.dependencies.repository.relation_builder import RelationBuilder
from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyInstallManager
from tool_shed.metadata import repository_metadata_manager
from tool_shed.util import basic_util
from tool_shed.util import commit_util
from tool_shed.util import common_util
from tool_shed.util import encoding_util
from tool_shed.util import hg_util
from tool_shed.util import metadata_util
from tool_shed.util import repository_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import xml_util
from tool_shed.util import (basic_util, commit_util, common_util, encoding_util,
hg_util, metadata_util, repository_util, shed_util_common as suc, xml_util)
log = logging.getLogger( __name__ )
@@ -822,7 +811,7 @@ class ImportRepositoryManager( object ):
if url:
valid_url = True
try:
stream = urllib.urlopen( url )
stream = urlopen( url )
except Exception as e:
valid_url = False
return_dict['error_message'] = 'Error importing file via http: %s' % str( e )
lib/tool_shed/dependencies/repository/relation_builder.py
+12 -13
View File
@@ -1,10 +1,9 @@
import logging
from galaxy.util import asbool, listify
import tool_shed.util.repository_util
from tool_shed.util import common_util, container_util, hg_util, metadata_util
from tool_shed.util import shed_util_common as suc
from galaxy.util import asbool, listify
from tool_shed.util import (common_util, container_util, hg_util, metadata_util,
shed_util_common as suc)
log = logging.getLogger( __name__ )
@@ -28,11 +27,11 @@ class RelationBuilder( object ):
def can_add_to_key_rd_dicts( self, key_rd_dict, key_rd_dicts ):
"""Handle the case where an update to the changeset revision was done."""
k = key_rd_dict.keys()[ 0 ]
k = next(iter(key_rd_dict))
rd = key_rd_dict[ k ]
partial_rd = rd[ 0:3 ]
for kr_dict in key_rd_dicts:
key = kr_dict.keys()[ 0 ]
key = next(iter(kr_dict))
if key == k:
repository_dependency = kr_dict[ key ]
if repository_dependency[ 0:3 ] == partial_rd:
@@ -175,7 +174,7 @@ class RelationBuilder( object ):
def get_updated_changeset_revisions_for_repository_dependencies( self, key_rd_dicts ):
updated_key_rd_dicts = []
for key_rd_dict in key_rd_dicts:
key = key_rd_dict.keys()[ 0 ]
key = next(iter(key_rd_dict))
repository_dependency = key_rd_dict[ key ]
rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \
rd_prior_installation_required, \
@@ -293,7 +292,7 @@ class RelationBuilder( object ):
def handle_next_repository_dependency( self ):
next_repository_key_rd_dict = self.key_rd_dicts_to_be_processed.pop( 0 )
next_repository_key_rd_dicts = [ next_repository_key_rd_dict ]
next_repository_key = next_repository_key_rd_dict.keys()[ 0 ]
next_repository_key = next(iter(next_repository_key_rd_dict))
self.handle_key_rd_dicts_for_repository( next_repository_key, next_repository_key_rd_dicts )
return self.get_repository_dependencies_for_changeset_revision()
@@ -326,7 +325,7 @@ class RelationBuilder( object ):
def in_key_rd_dicts( self, key_rd_dict, key_rd_dicts ):
"""Return True if key_rd_dict is contained in the list of key_rd_dicts."""
k = key_rd_dict.keys()[ 0 ]
k = next(iter(key_rd_dict))
v = key_rd_dict[ k ]
for key_rd_dict in key_rd_dicts:
for key, val in key_rd_dict.items():
@@ -436,11 +435,11 @@ class RelationBuilder( object ):
def remove_from_key_rd_dicts( self, key_rd_dict, key_rd_dicts ):
"""Eliminate the key_rd_dict from the list of key_rd_dicts if it is contained in the list."""
k = key_rd_dict.keys()[ 0 ]
k = next(iter(key_rd_dict))
v = key_rd_dict[ k ]
clean_key_rd_dicts = []
for krd_dict in key_rd_dicts:
key = krd_dict.keys()[ 0 ]
key = next(iter(krd_dict))
val = krd_dict[ key ]
if key == k and val == v:
continue
@@ -450,7 +449,7 @@ class RelationBuilder( object ):
def remove_repository_dependency_reference_to_self( self, key_rd_dicts ):
"""Remove all repository dependencies that point to a revision within its own repository."""
clean_key_rd_dicts = []
key = key_rd_dicts[ 0 ].keys()[ 0 ]
key = next(iter(key_rd_dicts[ 0 ]))
repository_tup = key.split( container_util.STRSEP )
rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \
rd_prior_installation_required, \
@@ -458,7 +457,7 @@ class RelationBuilder( object ):
common_util.parse_repository_dependency_tuple( repository_tup )
cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed )
for key_rd_dict in key_rd_dicts:
k = key_rd_dict.keys()[ 0 ]
k = next(iter(key_rd_dict))
repository_dependency = key_rd_dict[ k ]
toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( repository_dependency )
lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py
+1 -1
View File
@@ -91,7 +91,7 @@ class CustomDatatypeLoader( object ):
os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) )
os.write( fd, '</datatypes>\n' )
os.close( fd )
os.chmod( proprietary_datatypes_config, 0644 )
os.chmod( proprietary_datatypes_config, 0o644 )
# Load custom datatypes
self.app.datatypes_registry.load_datatypes( root_dir=self.app.config.root,
config=proprietary_datatypes_config,
lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py
+2 -3
View File
@@ -5,9 +5,8 @@ from sqlalchemy import false, or_
from galaxy import util
from galaxy.model import tool_shed_install
from galaxy.web import url_for
from galaxy.web.framework.helpers import iff, grids
from tool_shed.util import tool_dependency_util
from tool_shed.util import repository_util
from galaxy.web.framework.helpers import grids, iff
from tool_shed.util import repository_util, tool_dependency_util
log = logging.getLogger( __name__ )
lib/tool_shed/galaxy_install/install_manager.py
+5 -7
View File
@@ -11,10 +11,6 @@ from sqlalchemy import or_
from galaxy import exceptions, util
from galaxy.tools.deps import views
from tool_shed.util import basic_util, common_util, encoding_util, hg_util, repository_util
from tool_shed.util import shed_util_common as suc, tool_dependency_util
from tool_shed.util import tool_util, xml_util
from tool_shed.galaxy_install.datatypes import custom_datatype_manager
from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
@@ -23,8 +19,10 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment impor
from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager
from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager
from tool_shed.galaxy_install.tools import data_manager, tool_panel_manager
from tool_shed.tools import data_table_manager, tool_version_manager
from tool_shed.util import basic_util, common_util, encoding_util, hg_util, repository_util
from tool_shed.util import shed_util_common as suc, tool_dependency_util
from tool_shed.util import tool_util, xml_util
log = logging.getLogger( __name__ )
@@ -478,8 +476,8 @@ class InstallRepositoryManager( object ):
repository_revision_dict = items[ 1 ]
repo_info_dict = items[ 2 ]
else:
message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \
( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) )
message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s" % \
( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ) )
log.warning( message )
raise exceptions.InternalServerError( message )
# Make sure the tool shed returned everything we need for installing the repository.
lib/tool_shed/galaxy_install/installed_repository_manager.py
+6 -7
View File
@@ -8,6 +8,11 @@ import os
from sqlalchemy import and_, false, true
from galaxy import util
from tool_shed.galaxy_install.datatypes import custom_datatype_manager
from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
from tool_shed.galaxy_install.tools import data_manager
from tool_shed.galaxy_install.tools import tool_panel_manager
from tool_shed.util import common_util
from tool_shed.util import container_util
from tool_shed.util import repository_util
@@ -15,12 +20,6 @@ from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_dependency_util
from tool_shed.util import xml_util
from tool_shed.galaxy_install.datatypes import custom_datatype_manager
from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
from tool_shed.galaxy_install.tools import data_manager
from tool_shed.galaxy_install.tools import tool_panel_manager
log = logging.getLogger( __name__ )
@@ -242,7 +241,7 @@ class InstalledRepositoryManager( object ):
installed_td = {}
missing_rd = {}
missing_td = {}
name = repo_info_dict.keys()[ 0 ]
name = next(iter(repo_info_dict))
repo_info_tuple = repo_info_dict[ name ]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
repository_util.get_repo_info_tuple_contents( repo_info_tuple )
lib/tool_shed/galaxy_install/migrate/common.py
+7 -4
View File
@@ -1,9 +1,12 @@
import ConfigParser
from __future__ import print_function
import os
import sys
from six.moves import configparser
import galaxy.config
from tool_shed.galaxy_install import tool_migration_manager, installed_repository_manager
from tool_shed.galaxy_install import installed_repository_manager, tool_migration_manager
class MigrateToolsApplication( object, galaxy.config.ConfiguresGalaxyMixin ):
@@ -18,9 +21,9 @@ class MigrateToolsApplication( object, galaxy.config.ConfiguresGalaxyMixin ):
sys.argv.pop( pos )
galaxy_config_file = sys.argv.pop( pos )
if not os.path.exists( galaxy_config_file ):
print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % galaxy_config_file
print("Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % galaxy_config_file)
sys.exit( 1 )
config_parser = ConfigParser.ConfigParser( { 'here': os.getcwd() } )
config_parser = configparser.ConfigParser( { 'here': os.getcwd() } )
config_parser.read( galaxy_config_file )
galaxy_config_dict = {}
for key, value in config_parser.items( "app:main" ):
lib/tool_shed/galaxy_install/migrate/versions/0001_tools.py
+2 -1
View File
@@ -1,10 +1,11 @@
"""
Initialize the version column of the migrate_tools database table to 1. No tool migrations are handled in this version.
"""
from __future__ import print_function
def upgrade(migrate_engine):
print __doc__
print(__doc__)
def downgrade(migrate_engine):
lib/tool_shed/galaxy_install/migrate/versions/0002_tools.py
+2 -1
View File
@@ -4,10 +4,11 @@ datatypes_conf.xml.sample. You should remove the Emboss datatypes from your ver
repositories named emboss_5 and emboss_datatypes from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu
will be installed into your local Galaxy instance at the location discussed above by running the following command.
"""
from __future__ import print_function
def upgrade(migrate_engine):
print __doc__
print(__doc__)
def downgrade(migrate_engine):
lib/tool_shed/galaxy_install/migrate/versions/0003_tools.py
+2 -1
View File
@@ -3,10 +3,11 @@ The freebayes tool has been eliminated from the distribution . The repository n
Galaxy tool shed at http://toolshed.g2.bx.psu.edu will be installed into your local Galaxy instance at the
location discussed above by running the following command.
"""
from __future__ import print_function
def upgrade(migrate_engine):
print __doc__
print(__doc__)
def downgrade(migrate_engine):
lib/tool_shed/galaxy_install/migrate/versions/0004_tools.py
+2 -1
View File
@@ -5,10 +5,11 @@ blast_datatypes, in the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu.
These repositories will be installed into your local Galaxy instance at the
location discussed above by running the following command.
"""
from __future__ import print_function
def upgrade(migrate_engine):
print __doc__
print(__doc__)
def downgrade(migrate_engine):
lib/tool_shed/galaxy_install/migrate/versions/0005_tools.py
+2 -1
View File
@@ -6,10 +6,11 @@ http://toolshed.g2.bx.psu.edu, and will be installed into your local
Galaxy instance at the location discussed above by running the following
command.
"""
from __future__ import print_function
def upgrade(migrate_engine):
print __doc__
print(__doc__)
def downgrade(migrate_engine):
lib/tool_shed/galaxy_install/migrate/versions/0006_tools.py
+2 -1
View File
@@ -11,10 +11,11 @@ http://toolshed.g2.bx.psu.edu, and will be installed into your
local Galaxy instance at the location discussed above by running
the following command.
"""
from __future__ import print_function
def upgrade(migrate_engine):
print __doc__
print(__doc__)
def downgrade(migrate_engine):
lib/tool_shed/galaxy_install/migrate/versions/0007_tools.py
+2 -1
View File
@@ -8,10 +8,11 @@ http://toolshed.g2.bx.psu.edu, and will be installed into your
local Galaxy instance at the location discussed above by running
the following command.
"""
from __future__ import print_function
def upgrade(migrate_engine):
print __doc__
print(__doc__)
def downgrade(migrate_engine):
lib/tool_shed/galaxy_install/migrate/versions/0008_tools.py
+2 -1
View File
@@ -105,10 +105,11 @@ from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu
and will be installed into your local Galaxy instance at the
location discussed above by running the following command.
"""
from __future__ import print_function
def upgrade( migrate_engine ):
print __doc__
print(__doc__)
def downgrade( migrate_engine ):
lib/tool_shed/galaxy_install/migrate/versions/0009_tools.py
+2 -1
View File
@@ -95,10 +95,11 @@ from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu
and will be installed into your local Galaxy instance at the
location discussed above by running the following command.
"""
from __future__ import print_function
def upgrade( migrate_engine ):
print __doc__
print(__doc__)
def downgrade( migrate_engine ):
lib/tool_shed/galaxy_install/migrate/versions/0010_tools.py
+2 -1
View File
@@ -101,10 +101,11 @@ from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu
and will be installed into your local Galaxy instance at the
location discussed above by running the following command.
"""
from __future__ import print_function
def upgrade( migrate_engine ):
print __doc__
print(__doc__)
def downgrade( migrate_engine ):
lib/tool_shed/galaxy_install/migrate/versions/0011_tools.py
+2 -1
View File
@@ -53,10 +53,11 @@ from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu
and will be installed into your local Galaxy instance at the
location discussed above by running the following command.
"""
from __future__ import print_function
def upgrade( migrate_engine ):
print __doc__
print(__doc__)
def downgrade( migrate_engine ):
lib/tool_shed/galaxy_install/migrate/versions/0012_tools.py
+2 -1
View File
@@ -39,10 +39,11 @@ from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu
and will be installed into your local Galaxy instance at the
location discussed above by running the following command.
"""
from __future__ import print_function
def upgrade( migrate_engine ):
print __doc__
print(__doc__)
def downgrade( migrate_engine ):
lib/tool_shed/galaxy_install/repair_repository_manager.py
+6 -7
View File
@@ -1,20 +1,19 @@
import tempfile
import logging
log = logging.getLogger( __name__ )
import tempfile
from tool_shed.galaxy_install import install_manager
from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
from tool_shed.galaxy_install.tools import tool_panel_manager
from tool_shed.util import hg_util
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import container_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import hg_util
from tool_shed.util import repository_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_dependency_util
log = logging.getLogger( __name__ )
class RepairRepositoryManager():
@@ -118,7 +117,7 @@ class RepairRepositoryManager():
# Currently all tools contained within an installed tool shed repository must be loaded into the same
# section in the tool panel, so we can get the section id of the first guid in the tool_panel_section_dict.
# In the future, we'll have to handle different sections per guid.
guid = tool_panel_section_dict.keys()[ 0 ]
guid = next(iter(tool_panel_section_dict))
section_dicts = tool_panel_section_dict[ guid ]
section_dict = section_dicts[ 0 ]
tool_panel_section_id = section_dict[ 'id' ]
lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py
+9 -10
View File
@@ -5,13 +5,12 @@ into Galaxy from the Tool Shed.
import json
import logging
import os
import urllib
import urllib2
from galaxy.util import asbool, url_get, build_url
from six.moves.urllib.parse import urlencode
from six.moves.urllib.request import Request, urlopen
from galaxy.util import asbool, build_url, url_get
from tool_shed.galaxy_install.tools import tool_panel_manager
from tool_shed.util import common_util
from tool_shed.util import container_util
from tool_shed.util import encoding_util
@@ -380,10 +379,10 @@ class RepositoryDependencyInstallManager( object ):
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
pathspec = [ 'repository', 'get_required_repo_info_dict' ]
url = build_url( tool_shed_url, pathspec=pathspec )
# Fix for handling 307 redirect not being handled nicely by urllib2.urlopen when the urllib2.Request has data provided
url = urllib2.urlopen( urllib2.Request( url ) ).geturl()
request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
response = urllib2.urlopen( request ).read()
# Fix for handling 307 redirect not being handled nicely by urlopen() when the Request() has data provided
url = urlopen( Request( url ) ).geturl()
request = Request( url, data=urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
response = urlopen( request ).read()
if response:
try:
required_repo_info_dict = json.loads( response )
@@ -411,8 +410,8 @@ class RepositoryDependencyInstallManager( object ):
# lists of discovered repository dependencies, but these lists will be empty in the
# required_repo_info_dict since dependency discovery has not yet been performed for these
# dictionaries.
required_repo_info_dict_key = required_repo_info_dict.keys()[ 0 ]
all_repo_info_dicts_keys = [ d.keys()[ 0 ] for d in all_repo_info_dicts ]
required_repo_info_dict_key = next(iter(required_repo_info_dict))
all_repo_info_dicts_keys = [ next(iter(d)) for d in all_repo_info_dicts ]
if required_repo_info_dict_key not in all_repo_info_dicts_keys:
all_repo_info_dicts.append( required_repo_info_dict )
else:
lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py
+10 -10
View File
@@ -1,6 +1,5 @@
import logging
import os
import Queue
import shutil
import subprocess
import tempfile
@@ -9,8 +8,9 @@ import time
from contextlib import contextmanager
# TODO: eliminate the use of fabric here.
from fabric.operations import _AttributeString
from fabric import state
from fabric.operations import _AttributeString
from six.moves import queue
from galaxy.util import DATABASE_MAX_STRING_SIZE
from galaxy.util import DATABASE_MAX_STRING_SIZE_PRETTY
@@ -63,10 +63,10 @@ class InstallEnvironment( object ):
# Undoubtedly close() was called during a concurrent operation on the same file object.
log.debug( 'Error closing file descriptor: %s' % str( e ) )
time.sleep( .5 )
current_wait_time = time.time() - start_timer
if current_wait_time >= 600:
error = 'Error closing file descriptor: %s' % str( e )
break
current_wait_time = time.time() - start_timer
if current_wait_time >= 600:
error = 'Error closing file descriptor: %s' % str( e )
break
return error
def enqueue_output( self, stdout, stdout_queue, stderr, stderr_queue ):
@@ -169,10 +169,10 @@ class InstallEnvironment( object ):
cwd=state.env[ 'lcwd' ] )
pid = process_handle.pid
# Launch the asynchronous readers of the process' stdout and stderr.
stdout_queue = Queue.Queue()
stdout_queue = queue.Queue()
stdout_reader = asynchronous_reader.AsynchronousReader( process_handle.stdout, stdout_queue )
stdout_reader.start()
stderr_queue = Queue.Queue()
stderr_queue = queue.Queue()
stderr_reader = asynchronous_reader.AsynchronousReader( process_handle.stderr, stderr_queue )
stderr_reader.start()
# Place streamed stdout and stderr into a threaded IPC queue target so it can
@@ -192,7 +192,7 @@ class InstallEnvironment( object ):
while not stdout_queue.empty():
try:
line = stdout_queue.get()
except Queue.Empty:
except queue.Empty:
line = None
break
if line:
@@ -204,7 +204,7 @@ class InstallEnvironment( object ):
while not stderr_queue.empty():
try:
line = stderr_queue.get()
except Queue.Empty:
except queue.Empty:
line = None
break
if line:
lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
Executable → Regular
+6 -9
View File
@@ -1,26 +1,23 @@
import hashlib
import logging
import os
import re
import shutil
import stat
from string import Template
import tarfile
import tempfile
import time
import urllib2
import zipfile
import hashlib
from string import Template
# TODO: eliminate the use of fabric here.
from fabric.api import lcd, settings
from galaxy.util import asbool
from galaxy.util.template import fill_template
from tool_shed.util import basic_util
from tool_shed.util import tool_dependency_util
from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
# TODO: eliminate the use of fabric here.
from fabric.api import settings
from fabric.api import lcd
from tool_shed.util import basic_util, tool_dependency_util
log = logging.getLogger( __name__ )
lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
+3 -4
View File
@@ -4,6 +4,9 @@ import tempfile
from galaxy.tools.deps.resolvers import NullDependency
from galaxy.util import listify, url_get
from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder
from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import metadata_util
@@ -11,10 +14,6 @@ from tool_shed.util import repository_util
from tool_shed.util import tool_dependency_util
from tool_shed.util import xml_util
from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder
from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment
log = logging.getLogger( __name__ )
lib/tool_shed/galaxy_install/tool_migration_manager.py
+2 -2
View File
@@ -3,11 +3,11 @@ Manage automatic installation of tools configured in the xxx.xml files in ~/scri
All of the tools were at some point included in the Galaxy distribution, but are now hosted in the main Galaxy tool shed.
"""
import json
import logging
import os
import shutil
import tempfile
import threading
import logging
from galaxy import util
from galaxy.tools.toolbox import ToolSection
@@ -248,7 +248,7 @@ class ToolMigrationManager( object ):
tree.write( tmp_filename, encoding='utf-8', xml_declaration=True )
fh.close()
shutil.move( tmp_filename, os.path.abspath( proprietary_tool_conf ) )
os.chmod( proprietary_tool_conf, 0644 )
os.chmod( proprietary_tool_conf, 0o644 )
def get_containing_tool_sections( self, tool_config ):
"""
lib/tool_shed/galaxy_install/tools/data_manager.py
+2 -2
View File
@@ -124,7 +124,7 @@ class DataManagerHandler( object ):
if tree:
root = tree.getroot()
assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename )
guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ]
guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).values() if 'guid' in data_manager_dict ]
load_old_data_managers_by_guid = {}
data_manager_config_has_changes = False
config_elems = []
@@ -150,7 +150,7 @@ class DataManagerHandler( object ):
# Remove data managers from in memory
self.app.data_managers.remove_manager( guids )
# Load other versions of any now uninstalled data managers, if any
for elem in load_old_data_managers_by_guid.itervalues():
for elem in load_old_data_managers_by_guid.values():
self.app.data_managers.load_manager_from_elem( elem )
# Persist the altered shed_data_manager_config file.
if data_manager_config_has_changes:
lib/tool_shed/galaxy_install/tools/tool_panel_manager.py
+2 -2
View File
@@ -356,7 +356,7 @@ class ToolPanelManager( object ):
tool_panel_dict = self.generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
if tool_panel_dict:
# The tool_panel_dict is empty when tools exist but are not installed into a tool panel section.
tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ]
tool_section_dicts = tool_panel_dict[ next(iter(tool_panel_dict)) ]
tool_section_dict = tool_section_dicts[ 0 ]
original_section_id = tool_section_dict[ 'id' ]
if original_section_id:
@@ -423,7 +423,7 @@ class ToolPanelManager( object ):
self.app.install_model.context.flush()
# Create a list of guids for all tools that will be removed from the in-memory tool panel
# and config file on disk.
guids_to_remove = [ k for k in tool_panel_dict.keys() ]
guids_to_remove = list(tool_panel_dict.keys())
self.remove_guids( guids_to_remove, shed_tool_conf, uninstall )
def remove_guids( self, guids_to_remove, shed_tool_conf, uninstall ):
lib/tool_shed/galaxy_install/update_repository_manager.py
+1 -1
View File
@@ -7,10 +7,10 @@ import threading
from sqlalchemy import false
import tool_shed.util.shed_util_common as suc
from galaxy import util
from tool_shed.util import common_util
from tool_shed.util import encoding_util
from tool_shed.util import repository_util
from galaxy import util
log = logging.getLogger( __name__ )
lib/tool_shed/grids/admin_grids.py
+3 -6
View File
@@ -1,14 +1,11 @@
import logging
from markupsafe import escape
from sqlalchemy import and_
from markupsafe import escape
from galaxy.web.framework.helpers import grids
from galaxy.web.framework.helpers import time_ago
from galaxy.web.framework.helpers import grids, time_ago
from galaxy.webapps.tool_shed import model
from tool_shed.grids.repository_grids import CategoryGrid
from tool_shed.grids.repository_grids import RepositoryGrid
from tool_shed.grids.repository_grids import CategoryGrid, RepositoryGrid
from tool_shed.util import hg_util
log = logging.getLogger( __name__ )
lib/tool_shed/grids/repository_grids.py
+1 -1
View File
@@ -1140,7 +1140,7 @@ class ToolDependenciesGrid( RepositoryMetadataGrid ):
tds_dict = metadata.get( 'tool_dependencies', {} )
if tds_dict:
# Example: {"bwa/0.5.9": {"name": "bwa", "type": "package", "version": "0.5.9"}}
sorted_keys = sorted( [ k for k in tds_dict.keys() ] )
sorted_keys = sorted( tds_dict.keys() )
num_keys = len( sorted_keys )
# Handle environment settings first.
if 'set_environment' in sorted_keys:
lib/tool_shed/grids/repository_review_grids.py
+2 -4
View File
@@ -1,14 +1,12 @@
import logging
from sqlalchemy import and_, false, null, or_, true
from markupsafe import escape
from sqlalchemy import and_, false, null, or_, true
from galaxy.web.framework.helpers import grids
from galaxy.webapps.tool_shed import model
from tool_shed.grids.repository_grids import RepositoryGrid
from tool_shed.util import hg_util
from tool_shed.util import metadata_util
from tool_shed.util import hg_util, metadata_util
log = logging.getLogger( __name__ )
lib/tool_shed/managers/repositories.py
+6 -6
View File
@@ -1,13 +1,13 @@
"""
Manager and Serializer for TS repositories.
"""
from galaxy.exceptions import InconsistentDatabase
from galaxy.exceptions import RequestParameterInvalidException
from galaxy.exceptions import InternalServerError
from sqlalchemy.orm.exc import MultipleResultsFound
from sqlalchemy.orm.exc import NoResultFound
import logging
from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound
from galaxy.exceptions import (InconsistentDatabase, InternalServerError,
RequestParameterInvalidException)
log = logging.getLogger( __name__ )
lib/tool_shed/metadata/metadata_generator.py
+1 -1
View File
@@ -638,7 +638,7 @@ class MetadataGenerator( object ):
value, extra = required_file
required_files.append( ( value ) )
inputs = []
for param_name, values in ttb.inputs.iteritems():
for param_name, values in ttb.inputs.items():
# Handle improperly defined or strange test parameters and values.
if param_name is not None:
if values in [ None, False ]:
lib/tool_shed/metadata/repository_metadata_manager.py
+4 -10
View File
@@ -6,17 +6,11 @@ from sqlalchemy import false, or_
from galaxy import util
from galaxy.util import inflector
from galaxy.web.form_builder import SelectField
from tool_shed.metadata import metadata_generator
from tool_shed.repository_types.metadata import TipOnly
from tool_shed.repository_types import util as rt_util
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import hg_util
from tool_shed.util import metadata_util
from tool_shed.util import repository_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_util
from tool_shed.repository_types.metadata import TipOnly
from tool_shed.util import (basic_util, common_util, hg_util, metadata_util,
repository_util, shed_util_common as suc, tool_util)
log = logging.getLogger( __name__ )
@@ -175,7 +169,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
value.get( 'guid' ),
value.get( 'version' ),
value.get( 'name' ),
value.get( 'id' ) ) for name, value in metadata_dict.iteritems() ] ) )
value.get( 'id' ) ) for name, value in metadata_dict.items() ] ) )
# only compare valid entries, any invalid entries are ignored
ancestor_metadata = __data_manager_dict_to_tuple_list( ancestor_metadata.get( 'data_managers', {} ) )
current_metadata = __data_manager_dict_to_tuple_list( current_metadata.get( 'data_managers', {} ) )
lib/tool_shed/repository_types/registry.py
+5 -3
View File
@@ -1,9 +1,11 @@
import logging
import unrestricted
import repository_suite_definition
import tool_dependency_definition
from galaxy.util.odict import odict
from . import repository_suite_definition
from . import tool_dependency_definition
from . import unrestricted
log = logging.getLogger( __name__ )
lib/tool_shed/tool_shed_registry.py
+2 -1
View File
@@ -1,8 +1,9 @@
import logging
from six.moves.urllib import request as urlrequest
from galaxy.util.odict import odict
from tool_shed.util import common_util, xml_util
from six.moves.urllib import request as urlrequest
log = logging.getLogger( __name__ )
lib/tool_shed/tools/data_table_manager.py
+1 -1
View File
@@ -31,7 +31,7 @@ class ToolDataTableManager( object ):
changeset_revision_elem.text = changeset_revision
# add additional values
# TODO: enhance additional values to allow e.g. use of dict values that will recurse
for key, value in kwd.iteritems():
for key, value in kwd.items():
new_elem = XmlET.SubElement( elem, key )
new_elem.text = value
return elem
lib/tool_shed/util/hg_util.py
+1 -1
View File
@@ -201,7 +201,7 @@ def get_mercurial_default_options_dict( command, command_table=None, **kwd ):
possible = possible[0]
if len( possible ) != 1:
raise Exception('unable to find mercurial command "%s"' % command)
default_options_dict = dict( ( r[1].replace( '-', '_' ), r[2] ) for r in possible.values()[0][1][1] )
default_options_dict = dict( ( r[1].replace( '-', '_' ), r[2] ) for r in next(iter(possible.values()))[1][1] )
for option in kwd:
default_options_dict[ option ] = kwd[ option ]
return default_options_dict
lib/tool_shed/util/readme_util.py
+3 -3
View File
@@ -93,7 +93,7 @@ def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
selection page.
"""
name = repo_info_dict.keys()[ 0 ]
name = next(iter(repo_info_dict))
repo_info_tuple = repo_info_dict[ name ]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
repository_util.get_repo_info_tuple_contents( repo_info_tuple )
@@ -109,8 +109,8 @@ def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
def get_readme_file_names( repository_name ):
"""Return a list of file names that will be categorized as README files for the received repository_name."""
readme_files = [ 'readme', 'read_me', 'install' ]
valid_filenames = map( lambda f: '%s.txt' % f, readme_files )
valid_filenames.extend( map( lambda f: '%s.rst' % f, readme_files ) )
valid_filenames = ['%s.txt' % f for f in readme_files]
valid_filenames.extend( ['%s.rst' % f for f in readme_files] )
valid_filenames.extend( readme_files )
valid_filenames.append( '%s.txt' % repository_name )
valid_filenames.append( '%s.rst' % repository_name )
lib/tool_shed/util/repository_content_util.py
+1 -4
View File
@@ -1,11 +1,8 @@
import os
import shutil
from tool_shed.util import commit_util
from tool_shed.util import hg_util
from tool_shed.util import xml_util
import tool_shed.repository_types.util as rt_util
from tool_shed.util import commit_util, hg_util, xml_util
def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar,
lib/tool_shed/util/repository_util.py
+4 -7
View File
@@ -5,18 +5,15 @@ import shutil
from six.moves import configparser
from six.moves.urllib.error import HTTPError
from sqlalchemy import false, and_, or_
from sqlalchemy import and_, false, or_
import tool_shed.dependencies.repository
import tool_shed.util.metadata_util as metadata_util
from galaxy import util
from galaxy import web
from galaxy.web.form_builder import build_select_field
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import encoding_util
from tool_shed.util import hg_util
import tool_shed.util.metadata_util as metadata_util
from tool_shed.util import basic_util, common_util, encoding_util, hg_util
from tool_shed.util.web_util import escape
import tool_shed.dependencies.repository
log = logging.getLogger( __name__ )
lib/tool_shed/util/shed_util_common.py
+3 -5
View File
@@ -10,11 +10,9 @@ from sqlalchemy import and_, false, true
import tool_shed.util.repository_util
from galaxy import util
from galaxy.web import url_for
from galaxy.util import checkers
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import hg_util
from galaxy.web import url_for
from tool_shed.util import basic_util, common_util, hg_util
log = logging.getLogger( __name__ )
@@ -227,7 +225,7 @@ def get_unique_requirements(requirements):
if not type == "package":
continue
uniq_reqs["%s_%s" % (name, version)] = {'name': name, 'version': version, 'type': type}
return uniq_reqs.values()
return list(uniq_reqs.values())
def get_unique_requirements_from_repository(repository):
lib/tool_shed/util/tool_util.py
+1 -1
View File
@@ -246,7 +246,7 @@ def new_state( trans, tool, invalid=False ):
log.debug( 'Failed to build tool state for tool "%s" using standard method, will try to fall back on custom method: %s', tool.id, e )
inputs = tool.inputs_by_page[ 0 ]
context = ExpressionContext( state.inputs, parent=None )
for input in inputs.itervalues():
for input in inputs.values():
try:
state.inputs[ input.name ] = input.get_initial_value( trans, context )
except:
lib/tool_shed/util/web_util.py
+2 -2
View File
@@ -2,7 +2,7 @@ from markupsafe import escape as raw_escape
from galaxy.util import smart_str
ALLOWED_ELEMENTS = ["<b>", "</b>", "<br/>"]
ALLOWED_MAP = dict(map(lambda x: (x, raw_escape(x)), ALLOWED_ELEMENTS))
ALLOWED_MAP = dict((x, raw_escape(x)) for x in ALLOWED_ELEMENTS)
def escape( string ):
@@ -17,6 +17,6 @@ def escape( string ):
"""
escaped = smart_str( raw_escape( string ), encoding="ascii", errors="replace" )
# Unescape few selected tags.
for key, value in ALLOWED_MAP.iteritems():
for key, value in ALLOWED_MAP.items():
escaped = escaped.replace(value, key)
return escaped
lib/tool_shed/util/workflow_util.py
+3 -3
View File
@@ -95,7 +95,7 @@ class RepoToolModule( ToolModule ):
data_outputs = []
if self.tool:
data_inputs = None
for name, tool_output in self.tool.outputs.iteritems():
for name, tool_output in self.tool.outputs.items():
if tool_output.format_source is not None:
# Default to special name "input" which remove restrictions on connections
formats = [ 'input' ]
@@ -264,7 +264,7 @@ def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id,
# will be ( tool_id, tool_name, tool_version ).
missing_tool_tups = []
# First pass to build step objects and populate basic values
for step_dict in workflow_dict[ 'steps' ].itervalues():
for step_dict in workflow_dict[ 'steps' ].values():
# Create the model class for the step
step = trans.model.WorkflowStep()
step.label = step_dict.get('label', None)
@@ -302,7 +302,7 @@ def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id,
# Second pass to deal with connections between steps.
for step in steps:
# Input connections.
for input_name, conn_dict in step.temp_input_connections.iteritems():
for input_name, conn_dict in step.temp_input_connections.items():
if conn_dict:
output_step = steps_by_external_id[ conn_dict[ 'id' ] ]
conn = trans.model.WorkflowStepConnection()
lib/tool_shed/utility_containers/utility_container_manager.py
+1 -1
View File
@@ -212,7 +212,7 @@ class UtilityContainerManager( object ):
version='Version',
data_tables='Data Tables' )
folder.valid_data_managers.append( data_manager )
for data_manager_dict in data_managers.itervalues():
for data_manager_dict in data_managers.values():
data_manager_id += 1
try:
name = data_manager_dict.get( 'name', '' )
scripts/tool_shed/api/common.py
+1 -1
View File
@@ -6,8 +6,8 @@ import urllib2
sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir, os.pardir, 'lib' ) )
from tool_shed.util import hg_util
from galaxy import util
from tool_shed.util import hg_util
class HTTPRedirectWithDataHandler( urllib2.HTTPRedirectHandler ):
setup.cfg
+1 -1
View File
@@ -11,4 +11,4 @@ ignore = E128,E201,E202,E203,E501,E402,W503
# For flake8-import-order
# https://github.com/PyCQA/flake8-import-order/blob/master/tests/test_cases/complete_smarkets.py
import-order-style = smarkets
application-import-names = galaxy
application-import-names = galaxy,tool_shed