diff --git a/.ci/pep8_sources.txt b/.ci/pep8_sources.txt index 01ed8dca4cd..ce8e808d5e2 100644 --- a/.ci/pep8_sources.txt +++ b/.ci/pep8_sources.txt @@ -12,11 +12,7 @@ lib/galaxy/openid/__init__.py lib/galaxy/queues.py lib/galaxy/queue_worker.py lib/galaxy/tags -lib/galaxy/tools/deps/{commands,containers,dependencies,dockerfiles,docker_util,__init__,requirements}.py -lib/galaxy/tools/filters -lib/galaxy/tools/parser -lib/galaxy/tools/toolbox -lib/galaxy/tools/{errors,loader_directory,test}.py +lib/galaxy/tools lib/galaxy/util/{__init__,json,permutations,plugin_config,properties,simplegraph,sockets,sleeper,streamball,submodules}.py lib/galaxy/web/{proxy,security} lib/galaxy/web/base/{__init__,interactive_environments}.py @@ -33,6 +29,7 @@ lib/galaxy/webapps/tool_shed/framework/__init__.py lib/galaxy/webapps/tool_shed/framework/middleware/__init__.py lib/galaxy/webapps/tool_shed/util/{__init__,shed_statistics}.py lib/galaxy/work +lib/galaxy/workflow lib/galaxy/version.py lib/pulsar lib/tool_shed/__init__.py @@ -41,3 +38,6 @@ lib/tool_shed/galaxy_install/repository_dependencies lib/tool_shed/galaxy_install/tools/{__init__,tool_panel_manager}.py lib/tool_shed/repository_types/{__init__,registry,unrestricted}.py lib/tool_shed/util/{__init__,repository_content_util,tool_util,web_util,workflow_util}.py +test/functional/tools +tools/data_source/upload.py +tools/evolution/codingSnps_filter.py diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 9baa171ffb4..6a3bb3c2d1b 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -8,15 +8,11 @@ import json import logging import os import re -import shutil import threading -import types import urllib -import copy -import traceback from datetime import datetime -from galaxy import eggs, util +from galaxy import eggs eggs.require( "MarkupSafe" ) # MarkupSafe must load before mako eggs.require( "Mako" ) @@ -40,12 +36,10 @@ from galaxy.tools.deps import build_dependency_manager from galaxy.tools.parameters import params_to_incoming, check_param, params_from_strings, params_to_strings, visit_input_values from galaxy.tools.parameters import output_collect from galaxy.tools.parameters.basic import (BaseURLToolParameter, - DataToolParameter, DataCollectionToolParameter, HiddenToolParameter, LibraryDatasetToolParameter, - SelectToolParameter, ToolParameter, UnvalidatedValue, - IntegerToolParameter, FloatToolParameter) + DataToolParameter, DataCollectionToolParameter, HiddenToolParameter, + SelectToolParameter, ToolParameter, UnvalidatedValue) from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, Section, UploadDataset from galaxy.tools.parameters.input_translation import ToolInputTranslator -from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source @@ -55,7 +49,7 @@ from galaxy.util import rst_to_html, string_as_bool, string_to_object from galaxy.tools.parameters.meta import expand_meta_parameters from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext -from galaxy.util.hash_util import hmac_new, is_hashable +from galaxy.util.hash_util import hmac_new from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.web import url_for @@ -86,20 +80,21 @@ class ToolErrorLog: def __init__(self): self.error_stack = [] self.max_errors = 100 - + def add_error(self, file, phase, exception): self.error_stack.insert(0, { - "file" : file, - "time" : str(datetime.now()), - "phase" : phase, - "error" : str(exception) + "file": file, + "time": str(datetime.now()), + "phase": phase, + "error": str(exception) } ) if len(self.error_stack) > self.max_errors: self.error_stack.pop() - + global_tool_errors = ToolErrorLog() + class ToolNotFoundException( Exception ): pass @@ -126,7 +121,7 @@ class ToolBox( AbstractToolBox ): try: tool_source = get_tool_source( config_file, getattr( self.app.config, "enable_beta_tool_formats", False ) ) except Exception, e: - #capture and log parsing errors + # capture and log parsing errors global_tool_errors.add_error(config_file, "Tool XML parsing", e) raise e # Allow specifying a different tool subclass to instantiate @@ -448,7 +443,7 @@ class Tool( object, Dictifiable ): self.app = app self.repository_id = repository_id self._allow_code_files = allow_code_files - #setup initial attribute values + # setup initial attribute values self.inputs = odict() self.stdio_exit_codes = list() self.stdio_regexes = list() @@ -482,7 +477,7 @@ class Tool( object, Dictifiable ): self.version = None # Enable easy access to this tool's version lineage. self.lineage_ids = [] - #populate toolshed repository info, if available + # populate toolshed repository info, if available self.populate_tool_shed_info() # Parse XML element containing configuration try: @@ -534,7 +529,7 @@ class Tool( object, Dictifiable ): return False return True - return any( map( output_is_dynamic_collection, self.outputs.values() ) ) + return any( map( output_is_dynamic_collection, self.outputs.values() ) ) def __get_job_tool_configuration(self, job_params=None): """Generalized method for getting this tool's job configuration. @@ -1167,6 +1162,7 @@ class Tool( object, Dictifiable ): # TODO: Anyway to capture tools that dynamically change their own # outputs? return True + def new_state( self, trans, all_pages=False, history=None ): """ Create a new `DefaultToolState` for this tool. It will be initialized @@ -1183,6 +1179,7 @@ class Tool( object, Dictifiable ): inputs = self.inputs_by_page[ 0 ] self.fill_in_new_state( trans, inputs, state.inputs, history=history ) return state + def fill_in_new_state( self, trans, inputs, state, context=None, history=None ): """ Fill in a tool state dictionary with default values for all parameters @@ -1191,6 +1188,7 @@ class Tool( object, Dictifiable ): context = ExpressionContext( state, context ) for input in inputs.itervalues(): state[ input.name ] = input.get_initial_value( trans, context, history=history ) + def get_param_html_map( self, trans, page=0, other_values={} ): """ Return a dictionary containing the HTML representation of each @@ -1349,7 +1347,7 @@ class Tool( object, Dictifiable ): params = self.__remove_meta_properties( params ) job, out_data = self.execute( trans, incoming=params, history=history, rerun_remap_job_id=rerun_remap_job_id, mapping_over_collection=mapping_over_collection ) except httpexceptions.HTTPFound, e: - #if it's a paste redirect exception, pass it up the stack + # if it's a paste redirect exception, pass it up the stack raise e except Exception, e: log.exception('Exception caught while attempting tool execution:') @@ -1365,19 +1363,19 @@ class Tool( object, Dictifiable ): return False, message def __handle_state_refresh( self, trans, state, errors ): - try: - self.find_fieldstorage( state.inputs ) - except InterruptedUpload: - # If inputs contain a file it won't persist. Most likely this - # is an interrupted upload. We should probably find a more - # standard method of determining an incomplete POST. - return self.handle_interrupted( trans, state.inputs ) - except: - pass - # Just a refresh, render the form with updated state and errors. - if not self.display_interface: - return self.__no_display_interface_response() - return self.default_template, dict( errors=errors, tool_state=state ) + try: + self.find_fieldstorage( state.inputs ) + except InterruptedUpload: + # If inputs contain a file it won't persist. Most likely this + # is an interrupted upload. We should probably find a more + # standard method of determining an incomplete POST. + return self.handle_interrupted( trans, state.inputs ) + except: + pass + # Just a refresh, render the form with updated state and errors. + if not self.display_interface: + return self.__no_display_interface_response() + return self.default_template, dict( errors=errors, tool_state=state ) def __handle_page_advance( self, trans, state, errors ): state.page += 1 @@ -1404,7 +1402,7 @@ class Tool( object, Dictifiable ): def __check_param_values( self, trans, incoming, state, old_errors, process_state, history, source ): # Process incoming data - if not( self.check_values ): + if not self.check_values: # If `self.check_values` is false we don't do any checking or # processing on input This is used to pass raw values # through to/from external sites. FIXME: This should be handled @@ -1433,9 +1431,9 @@ class Tool( object, Dictifiable ): def find_fieldstorage( self, x ): if isinstance( x, FieldStorage ): raise InterruptedUpload( None ) - elif type( x ) is types.DictType: + elif isinstance(x, dict): [ self.find_fieldstorage( y ) for y in x.values() ] - elif type( x ) is types.ListType: + elif isinstance(x, list): [ self.find_fieldstorage( y ) for y in x ] def handle_interrupted( self, trans, inputs ): @@ -1544,8 +1542,7 @@ class Tool( object, Dictifiable ): history, source, prefix=group_prefix, - context=context, - ) + context=context) if group_errors: errors[ input.name ] = group_errors # Store the current case in a special value @@ -1572,7 +1569,7 @@ class Tool( object, Dictifiable ): any_group_errors = False d_type = input.get_datatype( trans, context ) writable_files = d_type.writable_files - #remove extra files + # remove extra files while len( group_state ) > len( writable_files ): del group_state[-1] @@ -1731,7 +1728,7 @@ class Tool( object, Dictifiable ): # are meant to be much more transient than the rest # of tool state. continue - #load default initial value + # load default initial value if not test_param_error: test_param_error = str( e ) if trans is not None: @@ -1798,7 +1795,7 @@ class Tool( object, Dictifiable ): any_group_errors = False d_type = input.get_datatype( trans, context ) writable_files = d_type.writable_files - #remove extra files + # remove extra files while len( group_state ) > len( writable_files ): del group_state[-1] if group_old_errors: @@ -2019,7 +2016,7 @@ class Tool( object, Dictifiable ): if allow_workflow_parameters and isinstance( values[ input.name ], basestring ): if WORKFLOW_PARAMETER_REGULAR_EXPRESSION.search( values[ input.name ] ): ck_param = False - #this will fail when a parameter's type has changed to a non-compatible one: e.g. conditional group changed to dataset input + # this will fail when a parameter's type has changed to a non-compatible one: e.g. conditional group changed to dataset input if ck_param: input.value_from_basic( input.value_to_basic( values[ input.name ], trans.app ), trans.app, ignore_errors=False ) except: @@ -2202,18 +2199,16 @@ class Tool( object, Dictifiable ): if 'job_working_directory' in self.app.config.collect_outputs_from: filenames.extend( glob.glob(os.path.join(job_working_directory, "child_%i_*" % outdata.id) ) ) for filename in filenames: - if not name in children: + if name not in children: children[name] = {} fields = os.path.basename(filename).split("_") - fields.pop(0) - parent_id = int(fields.pop(0)) - designation = fields.pop(0) - visible = fields.pop(0).lower() + designation = fields[2] + visible = fields[3].lower() if visible == "visible": visible = True else: visible = False - ext = fields.pop(0).lower() + ext = fields[4].lower() child_dataset = self.app.model.HistoryDatasetAssociation( extension=ext, parent_id=outdata.id, designation=designation, @@ -2297,7 +2292,7 @@ class Tool( object, Dictifiable ): return tool_dict - def to_json (self, trans, kwd={}, is_workflow=False): + def to_json(self, trans, kwd={}, is_workflow=False): """ Recursively creates a tool dictionary containing repeats, dynamic options and updated states. """ @@ -2308,7 +2303,7 @@ class Tool( object, Dictifiable ): # history id history = None try: - if not history_id is None: + if history_id is not None: history = self.history_manager.get_owned( trans.security.decode_id( history_id ), trans.user, current_history=trans.history ) else: history = trans.get_history() @@ -2352,8 +2347,8 @@ class Tool( object, Dictifiable ): tool_message = '' if job: try: - job_params = job.get_param_values( trans.app, ignore_errors = True ) - job_messages = self.check_and_update_param_values( job_params, trans, update_values=False ) + job_params = job.get_param_values( trans.app, ignore_errors=True ) + self.check_and_update_param_values( job_params, trans, update_values=False ) self._map_source_to_history( trans, self.inputs, job_params, history ) tool_message = self._compare_tool_version(trans, job) params_to_incoming( kwd, self.inputs, job_params, trans.app, to_html=False ) @@ -2363,7 +2358,7 @@ class Tool( object, Dictifiable ): return { 'error': str( exception ) } # create parameter object - params = galaxy.util.Params( kwd, sanitize = False ) + params = galaxy.util.Params( kwd, sanitize=False ) # convert value to jsonifiable value def jsonify(v): @@ -2488,7 +2483,7 @@ class Tool( object, Dictifiable ): group_state = state[input.name] = {} populate_state( trans, input.cases[current_case].inputs, group_state, errors, incoming, prefix=group_prefix, context=context) group_state['__current_case__'] = current_case - except Exception, e: + except Exception: errors[test_param_key] = 'The selected case is unavailable/invalid.' pass group_state[input.test_param.name] = value @@ -2526,7 +2521,7 @@ class Tool( object, Dictifiable ): test_param = tool_dict['test_param'] test_param['default_value'] = jsonify(input.test_param.get_initial_value(trans, other_values, history=history)) test_param['value'] = jsonify(group_state.get(test_param['name'], test_param['default_value'])) - for i in range (len ( tool_dict['cases'] ) ): + for i in range(len( tool_dict['cases'] ) ): current_state = {} if i == group_state.get('__current_case__', None): current_state = group_state @@ -2618,7 +2613,7 @@ class Tool( object, Dictifiable ): tool_versions = [] tools = self.app.toolbox.get_loaded_tools_by_lineage(self.id) for t in tools: - if not t.version in tool_versions: + if t.version not in tool_versions: tool_versions.append(t.version) # add information with underlying requirements and their versions @@ -2656,7 +2651,7 @@ class Tool( object, Dictifiable ): # return enriched tool model return tool_model - def _get_job_remap ( self, job): + def _get_job_remap( self, job): if job: if job.state == job.states.ERROR: try: @@ -2707,7 +2702,7 @@ class Tool( object, Dictifiable ): # Unpack unvalidated values to strings, they'll be validated when the # form is submitted (this happens when re-running a job that was # initially run by a workflow) - #This needs to be done recursively through grouping parameters + # This needs to be done recursively through grouping parameters def mapping_callback( input, value, prefixed_name, prefixed_label ): if isinstance( value, UnvalidatedValue ): try: @@ -2717,7 +2712,7 @@ class Tool( object, Dictifiable ): log.debug( "Failed to use input.to_html_value to determine value of unvalidated parameter, defaulting to string: %s" % ( e ) ) return str( value ) if isinstance( input, DataToolParameter ): - if isinstance(value,list): + if isinstance(value, list): values = [] for val in value: new_val = map_to_history( val ) @@ -2746,7 +2741,7 @@ class Tool( object, Dictifiable ): return { 'error': 'This dataset was created by an obsolete tool (%s). Can\'t re-run.' % tool_id } if ( self.id != tool_id and self.old_id != tool_id ) or self.version != tool_version: if self.id == tool_id: - if tool_version == None: + if tool_version is None: # for some reason jobs don't always keep track of the tool version. message = '' else: @@ -2764,12 +2759,12 @@ class Tool( object, Dictifiable ): message += 'currently not available. You can re-run the job with this tool, which is a derivation of the original tool.' except Exception, error: trans.response.status = 500 - return { 'error': str (error) } + return { 'error': str(error) } # can't rerun upload, external data sources, et cetera. workflow compatible will proxy this for now - #if not self.is_workflow_compatible: - # trans.response.status = 500 - # return { 'error': 'The \'%s\' tool does currently not support re-running.' % self.name } + # if not self.is_workflow_compatible: + # trans.response.status = 500 + # return { 'error': 'The \'%s\' tool does currently not support re-running.' % self.name } return message def get_default_history_by_trans( self, trans, create=False ): @@ -2831,9 +2826,9 @@ class OutputParameterJSONTool( Tool ): json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None for i, ( out_name, data ) in enumerate( out_data.iteritems() ): - #use wrapped dataset to access certain values + # use wrapped dataset to access certain values wrapped_data = param_dict.get( out_name ) - #allow multiple files to be created + # allow multiple files to be created file_name = str( wrapped_data ) extra_files_path = str( wrapped_data.files_path ) data_dict = dict( out_data_name=out_name, @@ -2883,9 +2878,9 @@ class DataSourceTool( OutputParameterJSONTool ): json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None for i, ( out_name, data ) in enumerate( out_data.iteritems() ): - #use wrapped dataset to access certain values + # use wrapped dataset to access certain values wrapped_data = param_dict.get( out_name ) - #allow multiple files to be created + # allow multiple files to be created cur_base_param_name = 'GALAXY|%s|' % out_name cur_name = param_dict.get( cur_base_param_name + 'name', name ) cur_dbkey = param_dict.get( cur_base_param_name + 'dkey', dbkey ) @@ -2991,12 +2986,12 @@ class DataManagerTool( OutputParameterJSONTool ): def exec_after_process( self, app, inp_data, out_data, param_dict, job=None, **kwds ): assert self.allow_user_access( job.user ), "You must be an admin to access this tool." - #run original exec_after_process + # run original exec_after_process super( DataManagerTool, self ).exec_after_process( app, inp_data, out_data, param_dict, job=job, **kwds ) - #process results of tool + # process results of tool if job and job.state == job.states.ERROR: return - #Job state may now be 'running' instead of previous 'error', but datasets are still set to e.g. error + # Job state may now be 'running' instead of previous 'error', but datasets are still set to e.g. error for dataset in out_data.itervalues(): if dataset.state != dataset.states.OK: return @@ -3017,7 +3012,7 @@ class DataManagerTool( OutputParameterJSONTool ): assert self.allow_user_access( user ), "You must be an admin to access this tool." history = user.data_manager_histories if not history: - #create + # create if create: history = _create_data_manager_history( user ) else: diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 1425cbc1946..93af340d2bd 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -1,8 +1,7 @@ from galaxy.exceptions import ObjectInvalid from galaxy.model import LibraryDatasetDatasetAssociation from galaxy import model -from galaxy.tools.parameters import DataToolParameter -from galaxy.tools.parameters import DataCollectionToolParameter +from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter from galaxy.tools.parameters.wrapped import WrappedParameters from galaxy.util import ExecutionTimer from galaxy.util.json import dumps @@ -51,7 +50,7 @@ class DefaultToolAction( object ): data = converted_dataset else: # FIXME: merge with hda.get_converted_dataset() mode as it's nearly identical. - #run converter here + # run converter here new_data = data.datatype.convert_dataset( trans, data, target_ext, return_output=True, visible=False ).values()[0] new_data.hid = data.hid new_data.name = data.name @@ -85,12 +84,12 @@ class DefaultToolAction( object ): if parent: parent[input.name][i] = input_datasets[ prefix + input.name + str( i + 1 ) ] for conversion_name, conversion_data in conversions: - #allow explicit conversion to be stored in job_parameter table + # allow explicit conversion to be stored in job_parameter table parent[ conversion_name ][i] = conversion_data.id # a more robust way to determine JSONable value is desired else: param_values[input.name][i] = input_datasets[ prefix + input.name + str( i + 1 ) ] for conversion_name, conversion_data in conversions: - #allow explicit conversion to be stored in job_parameter table + # allow explicit conversion to be stored in job_parameter table param_values[ conversion_name ][i] = conversion_data.id # a more robust way to determine JSONable value is desired else: input_datasets[ prefix + input.name ] = process_dataset( value ) @@ -107,7 +106,7 @@ class DefaultToolAction( object ): target_dict = param_values target_dict[ input.name ] = input_datasets[ prefix + input.name ] for conversion_name, conversion_data in conversions: - #allow explicit conversion to be stored in job_parameter table + # allow explicit conversion to be stored in job_parameter table target_dict[ conversion_name ] = conversion_data.id # a more robust way to determine JSONable value is desired elif isinstance( input, DataCollectionToolParameter ): if not value: @@ -121,7 +120,7 @@ class DefaultToolAction( object ): # some point and figure out if implicitly converting a # dataset collection makes senese. - #if i == 0: + # if i == 0: # # Allow copying metadata to output, first item will be source. # input_datasets[ prefix + input.name ] = data.dataset_instance input_datasets[ prefix + input.name + str( i + 1 ) ] = data @@ -229,8 +228,8 @@ class DefaultToolAction( object ): if output.parent: parent_to_child_pairs.append( ( output.parent, name ) ) child_dataset_names.add( name ) - ## What is the following hack for? Need to document under what - ## conditions can the following occur? (james@bx.psu.edu) + # What is the following hack for? Need to document under what + # conditions can the following occur? (james@bx.psu.edu) # HACK: the output data has already been created # this happens i.e. as a result of the async controller if name in incoming: @@ -276,7 +275,7 @@ class DefaultToolAction( object ): # Store output out_data[ name ] = data if output.actions: - #Apply pre-job tool-output-dataset actions; e.g. setting metadata, changing format + # Apply pre-job tool-output-dataset actions; e.g. setting metadata, changing format output_action_params = dict( out_data ) output_action_params.update( incoming ) output.actions.apply_action( data, output_action_params ) @@ -386,7 +385,7 @@ class DefaultToolAction( object ): for name, dataset in inp_data.iteritems(): if dataset: if not trans.app.security_agent.can_access_dataset( current_user_roles, dataset.dataset ): - raise "User does not have permission to use a dataset (%s) provided for input." % data.id + raise Exception("User does not have permission to use a dataset (%s) provided for input." % data.id) job.add_input_dataset( name, dataset ) else: job.add_input_dataset( name, None ) @@ -434,7 +433,7 @@ class DefaultToolAction( object ): trans.sa_session.add(jtid) jtod.dataset.visible = False trans.sa_session.add(jtod) - except Exception, e: + except Exception: log.exception('Cannot remap rerun dependencies.') trans.sa_session.flush() # Some tools are not really executable, but jobs are still created for them ( for record keeping ). @@ -554,7 +553,7 @@ def determine_output_format(output, parameter_context, input_datasets, random_in except Exception: pass - #process change_format tags + # process change_format tags if output.change_format is not None: new_format_set = False for change_elem in output.change_format: @@ -563,7 +562,7 @@ def determine_output_format(output, parameter_context, input_datasets, random_in if check is not None: try: if '$' not in check: - #allow a simple name or more complex specifications + # allow a simple name or more complex specifications check = '${%s}' % check if str( fill_template( check, context=parameter_context ) ) == when_elem.get( 'value', None ): ext = when_elem.get( 'format', ext ) diff --git a/lib/galaxy/tools/actions/data_manager.py b/lib/galaxy/tools/actions/data_manager.py index 15335b80dd6..cdf2ff8e61a 100644 --- a/lib/galaxy/tools/actions/data_manager.py +++ b/lib/galaxy/tools/actions/data_manager.py @@ -3,6 +3,7 @@ from __init__ import DefaultToolAction import logging log = logging.getLogger( __name__ ) + class DataManagerToolAction( DefaultToolAction ): """Tool action used for Data Manager Tools""" diff --git a/lib/galaxy/tools/actions/data_source.py b/lib/galaxy/tools/actions/data_source.py index c5ca41d1087..08086e9c93e 100644 --- a/lib/galaxy/tools/actions/data_source.py +++ b/lib/galaxy/tools/actions/data_source.py @@ -3,6 +3,7 @@ from __init__ import DefaultToolAction import logging log = logging.getLogger( __name__ ) + class DataSourceToolAction( DefaultToolAction ): """Tool action used for Data Source Tools""" diff --git a/lib/galaxy/tools/actions/history_imp_exp.py b/lib/galaxy/tools/actions/history_imp_exp.py index 3c425a29ba8..e6d3ff06cbc 100644 --- a/lib/galaxy/tools/actions/history_imp_exp.py +++ b/lib/galaxy/tools/actions/history_imp_exp.py @@ -2,7 +2,7 @@ import os import tempfile from galaxy.tools.actions import ToolAction -from galaxy.tools.imp_exp import JobExportHistoryArchiveWrapper, JobImportHistoryArchiveWrapper +from galaxy.tools.imp_exp import JobExportHistoryArchiveWrapper from galaxy.util.odict import odict import logging diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index 6acea06e90a..9bda45dbe95 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -6,7 +6,6 @@ from galaxy.util.odict import odict from galaxy.util.json import dumps from galaxy.jobs.datasets import DatasetPath - log = logging.getLogger( __name__ ) @@ -25,7 +24,7 @@ class SetMetadataToolAction( ToolAction ): return job, odict def execute_via_app( self, tool, app, session_id, history_id, user=None, - incoming = {}, set_output_hid = False, overwrite = True, + incoming={}, set_output_hid=False, overwrite=True, history=None, job_params=None ): """ Execute using application. @@ -55,18 +54,18 @@ class SetMetadataToolAction( ToolAction ): job.user_id = user.id if job_params: job.params = dumps( job_params ) - start_job_state = job.state #should be job.states.NEW + start_job_state = job.state # should be job.states.NEW try: # For backward compatibility, some tools may not have versions yet. job.tool_version = tool.version except: job.tool_version = "1.0.1" - job.state = job.states.WAITING #we need to set job state to something other than NEW, or else when tracking jobs in db it will be picked up before we have added input / output parameters + job.state = job.states.WAITING # we need to set job state to something other than NEW, or else when tracking jobs in db it will be picked up before we have added input / output parameters job.set_handler(tool.get_job_handler( job_params )) sa_session.add( job ) - sa_session.flush() #ensure job.id is available + sa_session.flush() # ensure job.id is available - #add parameters to job_parameter table + # add parameters to job_parameter table # Store original dataset state, so we can restore it. A separate table might be better (no chance of 'losing' the original state)? incoming[ '__ORIGINAL_DATASET_STATE__' ] = dataset.state input_paths = [DatasetPath( dataset.id, real_path=dataset.file_name, mutable=False )] @@ -75,36 +74,36 @@ class SetMetadataToolAction( ToolAction ): external_metadata_wrapper = JobExternalOutputMetadataWrapper( job ) cmd_line = external_metadata_wrapper.setup_external_metadata( dataset, sa_session, - exec_dir = None, - tmp_dir = job_working_dir, - dataset_files_path = app.model.Dataset.file_path, - output_fnames = input_paths, - config_root = app.config.root, - config_file = app.config.config_file, - datatypes_config = app.datatypes_registry.integrated_datatypes_configs, - job_metadata = None, - include_command = False, - kwds = { 'overwrite' : overwrite } ) + exec_dir=None, + tmp_dir=job_working_dir, + dataset_files_path=app.model.Dataset.file_path, + output_fnames=input_paths, + config_root=app.config.root, + config_file=app.config.config_file, + datatypes_config=app.datatypes_registry.integrated_datatypes_configs, + job_metadata=None, + include_command=False, + kwds={ 'overwrite' : overwrite } ) incoming[ '__SET_EXTERNAL_METADATA_COMMAND_LINE__' ] = cmd_line for name, value in tool.params_to_strings( incoming, app ).iteritems(): job.add_parameter( name, value ) - #add the dataset to job_to_input_dataset table + # add the dataset to job_to_input_dataset table if type == 'hda': job.add_input_dataset( dataset_name, dataset ) elif type == 'ldda': job.add_input_library_dataset( dataset_name, dataset ) - #Need a special state here to show that metadata is being set and also allow the job to run - # i.e. if state was set to 'running' the set metadata job would never run, as it would wait for input (the dataset to set metadata on) to be in a ready state + # Need a special state here to show that metadata is being set and also allow the job to run + # i.e. if state was set to 'running' the set metadata job would never run, as it would wait for input (the dataset to set metadata on) to be in a ready state dataset._state = dataset.states.SETTING_METADATA - job.state = start_job_state #job inputs have been configured, restore initial job state + job.state = start_job_state # job inputs have been configured, restore initial job state sa_session.flush() # Queue the job for execution app.job_queue.put( job.id, tool.id ) # FIXME: need to add event logging to app and log events there rather than trans. - #trans.log_event( "Added set external metadata job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) + # trans.log_event( "Added set external metadata job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) - #clear e.g. converted files + # clear e.g. converted files dataset.datatype.before_setting_metadata( dataset ) return job, odict() diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 447918d7a45..ea51d696ab6 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -6,7 +6,6 @@ import subprocess from cgi import FieldStorage from galaxy import datatypes, util from galaxy.util.odict import odict -from galaxy.datatypes import sniff from galaxy.util.json import dumps from galaxy.model.orm import eagerload_all from galaxy.exceptions import ObjectInvalid diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index fe965b6b575..aecb22488a0 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -139,7 +139,7 @@ class ToolDataTableManager( object ): if not remove_elems: remove_elems = [] full_path = os.path.abspath( shed_tool_data_table_config ) - #FIXME: we should lock changing this file by other threads / head nodes + # FIXME: we should lock changing this file by other threads / head nodes try: tree = util.parse_xml( full_path ) root = tree.getroot() @@ -148,10 +148,10 @@ class ToolDataTableManager( object ): out_elems = [] log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', e ) for elem in remove_elems: - #handle multiple occurrences of remove elem in existing elems + # handle multiple occurrences of remove elem in existing elems while elem in out_elems: remove_elems.remove( elem ) - #add new elems + # add new elems out_elems.extend( new_elems ) with open( full_path, 'wb' ) as out: out.write( '\n\n' ) @@ -273,7 +273,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): # Configure columns self.parse_column_spec( config_element ) - #store repo info if available: + # store repo info if available: repo_elem = config_element.find( 'tool_shed_repository' ) if repo_elem is not None: repo_info = dict( tool_shed=repo_elem.find( 'tool_shed' ).text, name=repo_elem.find( 'repository_name' ).text, @@ -303,7 +303,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): log.debug( "Encountered a file element (%s) that does not contain a path value when loading tool data table '%s'.", util.xml_to_string( file_element ), self.name ) continue - #FIXME: splitting on and merging paths from a configuration file when loading is wonky + # FIXME: splitting on and merging paths from a configuration file when loading is wonky # Data should exist on disk in the state needed, i.e. the xml configuration should # point directly to the desired file to load. Munging of the tool_data_tables_conf.xml.sample # can be done during installing / testing / metadata resetting with the creation of a proper @@ -349,18 +349,18 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): assert self.columns == other_table.columns, "Merging tabular data tables with non matching columns is not allowed: %s:%s != %s:%s" % ( self.name, self.columns, other_table.name, other_table.columns ) - #merge filename info + # merge filename info for filename, info in other_table.filenames.iteritems(): if filename not in self.filenames: self.filenames[ filename ] = info - #save info about table + # save info about table self._merged_load_info.append( ( other_table.__class__, other_table._load_info ) ) # If we are merging in a data table that does not allow duplicates, enforce that upon the data table if self.allow_duplicate_entries and not other_table.allow_duplicate_entries: log.debug( 'While attempting to merge tool data table "%s", the other instance of the table specified that duplicate entries are not allowed, now deduplicating all previous entries.', self.name ) self.allow_duplicate_entries = False self._deduplicate_data() - #add data entries and return current data table version + # add data entries and return current data table version return self.add_entries( other_table.data, allow_duplicates=allow_duplicates, persist=persist, persist_on_error=persist_on_error, entry_source=entry_source, **kwd ) def handle_found_index_file( self, filename ): @@ -469,7 +469,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): if not found_column: rval.append( name ) elif name == 'value': - #the column named 'value' always has priority over other named columns + # the column named 'value' always has priority over other named columns rval[ -1 ] = name found_column = True if not found_column: @@ -513,7 +513,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): def get_filename_for_source( self, source, default=None ): if source: - #if dict, assume is compatible info dict, otherwise call method + # if dict, assume is compatible info dict, otherwise call method if isinstance( source, dict ): source_repo_info = source else: @@ -529,7 +529,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): return filename def _add_entry( self, entry, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): - #accepts dict or list of columns + # accepts dict or list of columns if isinstance( entry, dict ): fields = [] for column_name in self.get_column_name_list(): @@ -557,11 +557,11 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): if persist and ( not is_error or persist_on_error ): filename = self.get_filename_for_source( entry_source ) if filename is None: - #should we default to using any filename here instead? + # should we default to using any filename here instead? log.error( "Unable to determine filename for persisting data table '%s' values: '%s'.", self.name, fields ) is_error = True else: - #FIXME: Need to lock these files for editing + # FIXME: Need to lock these files for editing log.debug( "Persisting changes to file: %s", filename ) try: data_table_fh = open( filename, 'r+b' ) @@ -587,18 +587,16 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): self.filter_file_fields( filename, values ) else: log.warn( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) ) - + self.reload_from_files() def filter_file_fields( self, loc_file, values ): """ Reads separated lines from file and print back only the lines that pass a filter. """ - separator_char = (lambda c: '' if c == '\t' else c)(self.separator) - with open(loc_file) as reader: rval = "" - for i, line in enumerate( reader ): + for line in reader: if line.lstrip().startswith( self.comment_char ): rval += line else: @@ -607,16 +605,16 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): fields = line_s.split( self.separator ) if fields != values: rval += line - + with open(loc_file, 'wb') as writer: writer.write(rval) - + return rval def _replace_field_separators( self, fields, separator=None, replace=None, comment_char=None ): - #make sure none of the fields contain separator - #make sure separator replace is different from comment_char, - #due to possible leading replace + # make sure none of the fields contain separator + # make sure separator replace is different from comment_char, + # due to possible leading replace if separator is None: separator = self.separator if replace is None: diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 36b45fc06af..96337874de6 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -10,7 +10,7 @@ from tool_shed.util import common_util import tool_shed.util.shed_util_common as suc import galaxy.queue_worker -#set up logger +# set up logger import logging log = logging.getLogger( __name__ ) @@ -38,7 +38,7 @@ class DataManagers( object ): tree = util.parse_xml( xml_filename ) except Exception, e: log.error( 'There was an error parsing your Data Manager config file "%s": %s' % ( xml_filename, e ) ) - return #we are not able to load any data managers + return # we are not able to load any data managers root = tree.getroot() if root.tag != 'data_managers': log.error( 'A data managers configuration must have a "data_managers" tag as the root. "%s" is present' % ( root.tag ) ) @@ -87,10 +87,10 @@ class DataManagers( object ): data_manager = self.get_manager( manager_id, None ) if data_manager is not None: del self.data_managers[ manager_id ] - #remove tool from toolbox + # remove tool from toolbox if data_manager.tool: self.app.toolbox.remove_tool_by_id( data_manager.tool.id ) - #determine if any data_tables are no longer tracked + # determine if any data_tables are no longer tracked for data_table_name in data_manager.data_tables.keys(): remove_data_table_tracking = True for other_data_manager in self.data_managers.itervalues(): @@ -135,7 +135,7 @@ class DataManager( object ): assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) ) path = tool_elem.get( "file", None ) tool_guid = tool_elem.get( "guid", None ) - #need to determine repository info so that dependencies will work correctly + # need to determine repository info so that dependencies will work correctly tool_shed_url = tool_elem.find( 'tool_shed' ).text # Handle protocol changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url ) @@ -156,7 +156,7 @@ class DataManager( object ): installed_changeset_revision ) if tool_shed_repository is None: log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' ) - #we'll set tool_path manually here from shed_conf_file + # we'll set tool_path manually here from shed_conf_file tool_shed_repository_id = None try: tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path ) @@ -164,7 +164,7 @@ class DataManager( object ): log.error( 'Error determining tool_path for Data Manager during testing: %s', e ) else: tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) - #use shed_conf_file to determine tool_path + # use shed_conf_file to determine tool_path shed_conf_file = elem.get( "shed_conf_file", None ) if shed_conf_file: shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None ) @@ -183,7 +183,7 @@ class DataManager( object ): data_table_name = data_table_elem.get( "name" ) assert data_table_name is not None, "A name is required for a data table entry" if data_table_name not in self.data_tables: - self.data_tables[ data_table_name ] = odict()#{} + self.data_tables[ data_table_name ] = odict() output_elem = data_table_elem.find( 'output' ) if output_elem is not None: for column_elem in output_elem.findall( 'column' ): @@ -217,7 +217,7 @@ class DataManager( object ): for move_elem in column_elem.findall( 'move' ): move_type = move_elem.get( 'type', 'directory' ) - relativize_symlinks = move_elem.get( 'relativize_symlinks', False ) #TODO: should we instead always relativize links? + relativize_symlinks = move_elem.get( 'relativize_symlinks', False ) # TODO: should we instead always relativize links? source_elem = move_elem.find( 'source' ) if source_elem is None: source_base = None @@ -244,7 +244,7 @@ class DataManager( object ): @property def id( self ): - return self.guid or self.declared_id #if we have a guid, we will use that as the data_manager id + return self.guid or self.declared_id # if we have a guid, we will use that as the data_manager id def load_tool( self, tool_filename, guid=None, data_manager_id=None, tool_shed_repository_id=None ): toolbox = self.data_managers.app.toolbox @@ -259,12 +259,12 @@ class DataManager( object ): def process_result( self, out_data ): data_manager_dicts = {} data_manager_dict = {} - #TODO: fix this merging below + # TODO: fix this merging below for output_name, output_dataset in out_data.iteritems(): try: output_dict = json.loads( open( output_dataset.file_name ).read() ) except Exception, e: - log.warning( 'Error reading DataManagerTool json for "%s": %s' % ( output_name, e ) ) + log.warning( 'Error reading DataManagerTool json for "%s": %s' % ( output_name, e ) ) continue data_manager_dicts[ output_name ] = output_dict for key, value in output_dict.iteritems(): @@ -274,18 +274,18 @@ class DataManager( object ): data_manager_dict.update( output_dict ) data_tables_dict = data_manager_dict.get( 'data_tables', {} ) - for data_table_name, data_table_columns in self.data_tables.iteritems(): + for data_table_name in self.data_tables.iterkeys(): data_table_values = data_tables_dict.pop( data_table_name, None ) if not data_table_values: log.warning( 'No values for data table "%s" were returned by the data manager "%s".' % ( data_table_name, self.id ) ) - continue #next data table + continue # next data table data_table = self.data_managers.app.tool_data_tables.get( data_table_name, None ) if data_table is None: log.error( 'The data manager "%s" returned an unknown data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( self.id, data_table_name, data_table_values, data_table_name, 'tool_data_table_conf.xml' ) ) - continue #next table name + continue # next table name if not isinstance( data_table, SUPPORTED_DATA_TABLE_TYPES ): log.error( 'The data manager "%s" returned an unsupported data table "%s" with type "%s" with new entries "%s". These entries will not be created. Please confirm that the data table is of a supported type (%s).' % ( self.id, data_table_name, type( data_table ), data_table_values, SUPPORTED_DATA_TABLE_TYPES ) ) - continue #next table name + continue # next table name output_ref_values = {} if data_table_name in self.output_ref_by_data_table: for data_table_column, output_ref in self.output_ref_by_data_table[ data_table_name ].iteritems(): @@ -296,10 +296,10 @@ class DataManager( object ): if not isinstance( data_table_values, list ): data_table_values = [ data_table_values ] for data_table_row in data_table_values: - data_table_value = dict( **data_table_row ) #keep original values here - for name, value in data_table_row.iteritems(): #FIXME: need to loop through here based upon order listed in data_manager config + data_table_value = dict( **data_table_row ) # keep original values here + for name, value in data_table_row.iteritems(): # FIXME: need to loop through here based upon order listed in data_manager config if name in output_ref_values: - moved = self.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value ) + self.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value ) data_table_value[ name ] = self.process_value_translation( data_table_name, name, **data_table_value ) data_table.add_entry( data_table_value, persist=True, entry_source=self ) galaxy.queue_worker.send_control_task(self.data_managers.app, 'reload_tool_data_tables', @@ -317,7 +317,7 @@ class DataManager( object ): if not isinstance( data_table_values, list ): data_table_values = [ data_table_values ] for data_table_row in data_table_values: - data_table_value = dict( **data_table_row ) #keep original values here + data_table_value = dict( **data_table_row ) # keep original values here for name, value in data_table_row.iteritems(): if name in path_column_names: data_table_value[ name ] = os.path.abspath( os.path.join( self.data_managers.app.config.galaxy_data_manager_data_path, value ) ) @@ -350,13 +350,13 @@ class DataManager( object ): target = os.path.join( target, fill_template( move_dict[ 'target_value' ], GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd ) ) if move_dict[ 'type' ] == 'file': - dirs, filename = os.path.split( target ) + dirs = os.path.split( target )[0] try: os.makedirs( dirs ) except OSError, e: if e.errno != errno.EEXIST: raise e - #moving a directory and the target already exists, we move the contents instead + # moving a directory and the target already exists, we move the contents instead util.move_merge( source, target ) if move_dict.get( 'relativize_symlinks', False ): diff --git a/lib/galaxy/tools/deps/brew_exts.py b/lib/galaxy/tools/deps/brew_exts.py index ca0cd803980..9746bfcfdc3 100755 --- a/lib/galaxy/tools/deps/brew_exts.py +++ b/lib/galaxy/tools/deps/brew_exts.py @@ -438,7 +438,7 @@ def execute(cmds, env=None): if env: subprocess_kwds["env"] = env p = subprocess.Popen(cmds, **subprocess_kwds) - #log = p.stdout.read() + # log = p.stdout.read() global VERBOSE stdout, stderr = p.communicate() if p.returncode != 0: diff --git a/lib/galaxy/tools/deps/brew_util.py b/lib/galaxy/tools/deps/brew_util.py index 85ecf5c8911..b970af725f8 100644 --- a/lib/galaxy/tools/deps/brew_util.py +++ b/lib/galaxy/tools/deps/brew_util.py @@ -37,4 +37,3 @@ def requirements_to_recipe_contexts(requirements, brew_context): brew_context ) return map(to_recipe_context, requirements_to_recipes(requirements)) - diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index 18e2ffe2790..3d190f85bd6 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -11,11 +11,11 @@ class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirM resolver_type = "galaxy_packages" def __init__(self, dependency_manager, **kwds): - ## Galaxy tool shed requires explicit versions on XML elements, - ## this in inconvient for testing or Galaxy instances not utilizing - ## the tool shed so allow a fallback version of the Galaxy package - ## resolver that will just grab 'default' version of exact version - ## unavailable. + # Galaxy tool shed requires explicit versions on XML elements, + # this in inconvient for testing or Galaxy instances not utilizing + # the tool shed so allow a fallback version of the Galaxy package + # resolver that will just grab 'default' version of exact version + # unavailable. self.versionless = str(kwds.get('versionless', "false")).lower() == "true" self._init_base_path( dependency_manager, **kwds ) diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index a07edcdb7f2..b08c255c7f0 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -70,7 +70,6 @@ class DirectoryModuleChecker(object): self.directories = modulepath.split(pathsep) if prefetch: log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") - pass def has_module(self, module, version): has_module = False @@ -107,7 +106,7 @@ class AvailModuleChecker(object): for module_name, module_version in module_generator: names_match = module == module_name - module_match = names_match and (version == None or module_version == version) + module_match = names_match and (version is None or module_version == version) if module_match: return True return False diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 8dc7b573788..730499ca919 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -164,11 +164,11 @@ class ToolEvaluator( object ): tool=self.tool, name=input.name ) elif isinstance( input, DataToolParameter ): - ## FIXME: We're populating param_dict with conversions when - ## wrapping values, this should happen as a separate - ## step before wrapping (or call this wrapping step - ## something more generic) (but iterating this same - ## list twice would be wasteful) + # FIXME: We're populating param_dict with conversions when + # wrapping values, this should happen as a separate + # step before wrapping (or call this wrapping step + # something more generic) (but iterating this same + # list twice would be wasteful) # Add explicit conversions by name to current parent for conversion_name, conversion_extensions, conversion_datatypes in input.conversions: # If we are at building cmdline step, then converters @@ -205,7 +205,7 @@ class ToolEvaluator( object ): if identifier_key in param_dict: wrapper_kwds["identifier"] = param_dict[identifier_key] if dataset: - #A None dataset does not have a filename + # A None dataset does not have a filename real_path = dataset.file_name if real_path in input_dataset_paths: wrapper_kwds[ "dataset_path" ] = input_dataset_paths[ real_path ] @@ -240,16 +240,16 @@ class ToolEvaluator( object ): def __populate_input_dataset_wrappers(self, param_dict, input_datasets, input_dataset_paths): # TODO: Update this method for dataset collections? Need to test. -John. - ## FIXME: when self.check_values==True, input datasets are being wrapped - ## twice (above and below, creating 2 separate - ## DatasetFilenameWrapper objects - first is overwritten by - ## second), is this necessary? - if we get rid of this way to - ## access children, can we stop this redundancy, or is there - ## another reason for this? - ## - Only necessary when self.check_values is False (==external dataset - ## tool?: can this be abstracted out as part of being a datasouce tool?) - ## - But we still want (ALWAYS) to wrap input datasets (this should be - ## checked to prevent overhead of creating a new object?) + # FIXME: when self.check_values==True, input datasets are being wrapped + # twice (above and below, creating 2 separate + # DatasetFilenameWrapper objects - first is overwritten by + # second), is this necessary? - if we get rid of this way to + # access children, can we stop this redundancy, or is there + # another reason for this? + # - Only necessary when self.check_values is False (==external dataset + # tool?: can this be abstracted out as part of being a datasouce tool?) + # - But we still want (ALWAYS) to wrap input datasets (this should be + # checked to prevent overhead of creating a new object?) # Additionally, datasets go in the param dict. We wrap them such that # if the bare variable name is used it returns the filename (for # backwards compatibility). We also add any child datasets to the @@ -281,9 +281,9 @@ class ToolEvaluator( object ): for name, out_collection in output_collections.items(): if name not in tool.output_collections: continue - #message_template = "Name [%s] not found in tool.output_collections %s" - #message = message_template % ( name, tool.output_collections ) - #raise AssertionError( message ) + # message_template = "Name [%s] not found in tool.output_collections %s" + # message = message_template % ( name, tool.output_collections ) + # raise AssertionError( message ) wrapper_kwds = dict( datatypes_registry=self.app.datatypes_registry, @@ -371,7 +371,7 @@ class ToolEvaluator( object ): if not self.tool.check_values and self.unstructured_path_rewriter: # The tools weren't "wrapped" yet, but need to be in order to get - #the paths rewritten. + # the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) def __sanitize_param_dict( self, param_dict ): @@ -379,7 +379,7 @@ class ToolEvaluator( object ): Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, which already have their own specific sanitization rules and also exclude special-cased named values. We will only examine the first level for values to skip; the wrapping function will recurse as necessary. - + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. """ # chromInfo is a filename, do not sanitize it. @@ -404,22 +404,22 @@ class ToolEvaluator( object ): try: self.__build_config_files( ) except Exception, e: - #capture and log parsing errors + # capture and log parsing errors global_tool_errors.add_error(self.tool.config_file, "Building Config Files", e) - raise e + raise e try: self.__build_param_file( ) except Exception, e: - #capture and log parsing errors + # capture and log parsing errors global_tool_errors.add_error(self.tool.config_file, "Building Param File", e) raise e try: self.__build_command_line( ) except Exception, e: - #capture and log parsing errors + # capture and log parsing errors global_tool_errors.add_error(self.tool.config_file, "Building Command Line", e) raise e - + return self.command_line, self.extra_filenames def __build_command_line( self ): @@ -444,7 +444,7 @@ class ToolEvaluator( object ): command_line = command_line.replace( "\n", " " ).replace( "\r", " " ).strip() except Exception: # Modify exception message to be more clear - #e.args = ( 'Error substituting into command line. Params: %r, Command: %s' % ( param_dict, self.command ), ) + # e.args = ( 'Error substituting into command line. Params: %r, Command: %s' % ( param_dict, self.command ), ) raise if interpreter: # TODO: path munging for cluster/dataset server relocatability @@ -469,7 +469,7 @@ class ToolEvaluator( object ): else: fd, config_filename = tempfile.mkstemp( dir=directory ) os.close( fd ) - f = open( config_filename, "wt" ) + f = open( config_filename, "w" ) f.write( fill_template( template_text, context=param_dict ) ) f.close() # For running jobs as the actual user, ensure the config file is globally readable @@ -488,10 +488,10 @@ class ToolEvaluator( object ): if command and "$param_file" in command: fd, param_filename = tempfile.mkstemp( dir=directory ) os.close( fd ) - f = open( param_filename, "wt" ) + f = open( param_filename, "w" ) for key, value in param_dict.items(): # parameters can be strings or lists of strings, coerce to list - if type(value) != type([]): + if not isinstance(value, list): value = [ value ] for elem in value: f.write( '%s=%s\n' % (key, elem) ) diff --git a/lib/galaxy/tools/exception_handling.py b/lib/galaxy/tools/exception_handling.py index f65b9a09469..ad62ea763b2 100644 --- a/lib/galaxy/tools/exception_handling.py +++ b/lib/galaxy/tools/exception_handling.py @@ -5,17 +5,21 @@ FIXME: These are used by tool scripts, not the framework, and should not live in this package. """ + class UCSCLimitException( Exception ): pass + class UCSCOutWrapper( object ): """File-like object that throws an exception if it encounters the UCSC limit error lines""" def __init__( self, other ): self.other = iter( other ) # Need one line of lookahead to be sure we are hitting the limit message self.lookahead = None + def __iter__( self ): return self + def next( self ): if self.lookahead is None: line = self.other.next() @@ -29,5 +33,6 @@ class UCSCOutWrapper( object ): else: self.lookahead = next_line return line + def readline(self): return self.next() diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index 9b5f79b326e..d7456617196 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -88,9 +88,7 @@ class ToolExecutionTracker( object ): # collection replaced with a specific dataset. Need to replace this # with the collection and wrap everything up so can evaluate output # label. - params.update( self.collection_info.collections ) # Replace datasets - # with source collections - # for labelling outputs. + params.update( self.collection_info.collections ) # Replace datasets with source collections for labelling outputs. collection_names = map( lambda c: "collection %d" % c.hid, collections ) on_text = on_text_for_names( collection_names ) diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index 5f454b1b795..f8af61d9124 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -11,7 +11,6 @@ import os import sys import tarfile -from galaxy import eggs from galaxy.util.json import dumps, loads diff --git a/lib/galaxy/tools/linters/citations.py b/lib/galaxy/tools/linters/citations.py index 2ff7e8c4cb1..aaa468fad50 100644 --- a/lib/galaxy/tools/linters/citations.py +++ b/lib/galaxy/tools/linters/citations.py @@ -24,4 +24,3 @@ def lint_citations(tool_xml, lint_ctx): if valid_citations > 0: lint_ctx.valid("Found %d likely valid citations.", valid_citations) - diff --git a/lib/galaxy/tools/loader.py b/lib/galaxy/tools/loader.py index 42fc53105ba..bb2c5ce802a 100644 --- a/lib/galaxy/tools/loader.py +++ b/lib/galaxy/tools/loader.py @@ -66,8 +66,8 @@ def _macros_of_type(root, type, el_func): macro_dict = {} if macros_el is not None: macro_els = macros_el.findall('macro') - macro_dict = dict([(macro_el.get("name"), el_func(macro_el)) \ - for macro_el in macro_els \ + macro_dict = dict([(macro_el.get("name"), el_func(macro_el)) + for macro_el in macro_els if macro_el.get('type') == type]) return macro_dict @@ -80,11 +80,11 @@ def _expand_tokens(elements, tokens): value = element.text if value: new_value = _expand_tokens_str(element.text, tokens) - if not (new_value is value): + if new_value is not value: element.text = new_value for key, value in element.attrib.iteritems(): new_value = _expand_tokens_str(value, tokens) - if not (new_value is value): + if new_value is not value: element.attrib[key] = new_value _expand_tokens(list(element), tokens) @@ -211,10 +211,10 @@ def _xml_set_children(element, new_children): def _xml_replace(query, targets, parent_map): - #parent_el = query.find('..') ## Something like this would be better with newer xml library + # parent_el = query.find('..') ## Something like this would be better with newer xml library parent_el = parent_map[query] matching_index = -1 - #for index, el in enumerate(parent_el.iter('.')): ## Something like this for newer implementation + # for index, el in enumerate(parent_el.iter('.')): ## Something like this for newer implementation for index, el in enumerate(list(parent_el)): if el == query: matching_index = index diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index a25f2cf7e19..8190cd8d89b 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -2,9 +2,9 @@ Classes encapsulating Galaxy tool parameters. """ -from basic import * -from grouping import * -from galaxy.util.json import * +from basic import DataCollectionToolParameter, DataToolParameter, SelectToolParameter +from grouping import Conditional, Repeat, Section, UploadDataset +from galaxy.util.json import dumps, json_fix, loads def visit_input_values( inputs, input_values, callback, name_prefix="", label_prefix="" ): @@ -119,7 +119,7 @@ def params_to_incoming( incoming, inputs, input_values, app, name_prefix="", to_ """ for input in inputs.itervalues(): if isinstance( input, Repeat ) or isinstance( input, UploadDataset ): - for i, d in enumerate( input_values[ input.name ] ): + for d in input_values[ input.name ]: index = d['__index__'] new_name_prefix = name_prefix + "%s_%d|" % ( input.name, index ) params_to_incoming( incoming, input.inputs, d, app, new_name_prefix, to_html=to_html) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index e316e5bca31..b00cc86c828 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -3,22 +3,18 @@ Basic tool parameters. """ import logging -import string -import sys import os import os.path import urllib from xml.etree.ElementTree import XML -from galaxy import config, datatypes, util +from galaxy import util from galaxy.web import form_builder -from galaxy.util.bunch import Bunch from galaxy.util import string_as_bool, sanitize_param, unicodify from galaxy.util import listify from galaxy.util.odict import odict from galaxy.util.expressions import ExpressionContext from sanitize import ToolParameterSanitizer import validation -import dynamic_options import galaxy.tools.parser from ..parser import get_input_source as ensure_input_source from ..parameters import history_query @@ -209,11 +205,11 @@ class ToolParameter( object, Dictifiable ): def to_dict( self, trans, view='collection', value_mapper=None, other_values={} ): """ to_dict tool parameter. This can be overridden by subclasses. """ tool_dict = super( ToolParameter, self ).to_dict() - #TODO: wrapping html as it causes a lot of errors on subclasses - needs histories, etc. + # TODO: wrapping html as it causes a lot of errors on subclasses - needs histories, etc. try: tool_dict[ 'html' ] = urllib.quote( util.smart_str( self.get_html( trans ) ) ) - except AssertionError, e: - pass #HACK for assert trans.history, 'requires a history' + except AssertionError: + pass # HACK for assert trans.history, 'requires a history' tool_dict[ 'model_class' ] = self.__class__.__name__ tool_dict[ 'optional' ] = self.optional @@ -296,6 +292,7 @@ class TextToolParameter( ToolParameter ): d['size'] = self.size return d + class IntegerToolParameter( TextToolParameter ): """ Parameter that takes an integer value. @@ -594,8 +591,8 @@ class FTPFileToolParameter( ToolParameter ): self.user_ftp_dir = '' def get_initial_value( self, trans, context, history=None ): - if not trans is None: - if not trans.user is None: + if trans is not None: + if trans.user is not None: self.user_ftp_dir = "%s/" % trans.user_ftp_dir return None @@ -652,6 +649,7 @@ class FTPFileToolParameter( ToolParameter ): d['multiple'] = self.multiple return d + class HiddenToolParameter( ToolParameter ): """ Parameter that takes one of two values. @@ -698,7 +696,7 @@ class ColorToolParameter( ToolParameter ): return form_builder.HiddenField( self.name, self.value ) def get_initial_value( self, trans, context, history=None ): - return self.value.lower(); + return self.value.lower() class BaseURLToolParameter( HiddenToolParameter ): @@ -1188,7 +1186,7 @@ class ColumnListParameter( SelectToolParameter ): removes the 'c' when entered into a workflow. """ if self.multiple: - #split on newline and , + # split on newline and , if isinstance( value, list ) or isinstance( value, basestring ): column_list = [] if not isinstance( value, list ): @@ -1271,28 +1269,27 @@ class ColumnListParameter( SelectToolParameter ): """ show column labels rather than c1..cn if use_header_names=True """ options = [] - colnames = None - if self.usecolnames: # read first row - assume is a header with metadata useful for making good choices + if self.usecolnames: # read first row - assume is a header with metadata useful for making good choices assert self.data_ref in other_values, "Value for associated DataToolParameter not found" dataset = other_values[ self.data_ref ] try: - head = open(dataset.get_file_name(),'r').readline() + head = open(dataset.get_file_name(), 'r').readline() cnames = head.rstrip().split('\t') - column_list = [('%d' % (i+1),'c%d: %s' % (i+1,x)) for i,x in enumerate(cnames)] - if self.numerical: # If numerical was requested, filter columns based on metadata + column_list = [('%d' % (i + 1), 'c%d: %s' % (i + 1, x)) for i, x in enumerate(cnames)] + if self.numerical: # If numerical was requested, filter columns based on metadata if len(dataset.metadata.column_types) >= len(cnames): - numerics = [i for i,x in enumerate(dataset.metadata.column_types) if x == 'int' or x == 'float'] + numerics = [i for i, x in enumerate(dataset.metadata.column_types) if x == 'int' or x == 'float'] column_list = [column_list[i] for i in numerics] except: column_list = self.get_column_list( trans, other_values ) else: - column_list = self.get_column_list( trans, other_values ) + column_list = self.get_column_list( trans, other_values ) if len( column_list ) > 0 and not self.force_select: options.append( ('?', 'None', False) ) for col in column_list: if col != 'None': - if type(col) == type(()) and len(col) == 2: # fiddled - options.append((col[1],col[0],False)) + if isinstance(col, tuple) and len(col) == 2: # fiddled + options.append((col[1], col[0], False)) else: options.append( ( 'Column: ' + col, col, False ) ) return options @@ -1455,7 +1452,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): if elem.find( 'filter' ): self.is_dynamic = True for filter in elem.findall( 'filter' ): - #currently only filtering by metadata key matching input file is allowed + # currently only filtering by metadata key matching input file is allowed if filter.get( 'type' ) == 'data_meta': if filter.get( 'data_ref' ) not in self.filtered: self.filtered[filter.get( 'data_ref' )] = {} @@ -1585,7 +1582,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): options = get_options_list( val ) rval.extend( options ) if len( rval ) > 1: - if not( self.repeat ): + if not self.repeat: assert self.multiple, "Multiple values provided but parameter is not expecting multiple values" rval = self.separator.join( map( value_map, rval ) ) if self.tool is None or self.tool.options.sanitize: @@ -1706,7 +1703,7 @@ class BaseDataToolParameter( ToolParameter ): # A handle to the transaction (and thus app) will be given by the module. datatypes_registry = trans.app.datatypes_registry else: - #This occurs for things such as unit tests + # This occurs for things such as unit tests import galaxy.datatypes.registry datatypes_registry = galaxy.datatypes.registry.Registry() datatypes_registry.load_datatypes() @@ -1919,7 +1916,7 @@ class DataToolParameter( BaseDataToolParameter ): most_recent_dataset = [] def dataset_collector( datasets ): - for i, data in enumerate( datasets ): + for data in datasets: if data.visible and dataset_matcher.hda_accessible( data, check_security=False ): match = dataset_matcher.valid_hda_match( data, check_security=False ) if not match or dataset_matcher.filter( match.hda ): @@ -2099,9 +2096,9 @@ class DataToolParameter( BaseDataToolParameter ): return allow def _options_filter_attribute( self, value ): - #HACK to get around current hardcoded limitation of when a set of dynamic options is defined for a DataToolParameter - #it always causes available datasets to be filtered by dbkey - #this behavior needs to be entirely reworked (in a backwards compatible manner) + # HACK to get around current hardcoded limitation of when a set of dynamic options is defined for a DataToolParameter + # it always causes available datasets to be filtered by dbkey + # this behavior needs to be entirely reworked (in a backwards compatible manner) options_filter_attribute = self.options_filter_attribute if options_filter_attribute is None: return value.get_dbkey() @@ -2361,22 +2358,22 @@ class DataCollectionToolParameter( BaseDataToolParameter ): # append directly matched collections for hdca in self.match_collections( trans, history, dataset_matcher ): d['options']['hdca'].append({ - 'id' : trans.security.encode_id( hdca.id ), - 'hid' : hdca.hid, - 'name' : hdca.name, - 'src' : 'hdca' - }) + 'id': trans.security.encode_id( hdca.id ), + 'hid': hdca.hid, + 'name': hdca.name, + 'src': 'hdca' + }) # append matching subcollections for hdca in self.match_multirun_collections( trans, history, dataset_matcher ): subcollection_type = self._history_query( trans ).collection_type_description.collection_type d['options']['hdca'].append({ - 'id' : trans.security.encode_id( hdca.id ), - 'hid' : hdca.hid, - 'name' : hdca.name, - 'src' : 'hdca', - 'map_over_type' : subcollection_type - }) + 'id': trans.security.encode_id( hdca.id ), + 'hid': hdca.hid, + 'name': hdca.name, + 'src': 'hdca', + 'map_over_type': subcollection_type + }) # sort both lists d['options']['hdca'] = sorted(d['options']['hdca'], key=lambda k: k['hid'], reverse=True) @@ -2438,10 +2435,10 @@ class LibraryDatasetToolParameter( ToolParameter ): lst = [] for item in value: encoded_id = encoded_name = None - if isinstance (item, app.model.LibraryDatasetDatasetAssociation): + if isinstance(item, app.model.LibraryDatasetDatasetAssociation): encoded_id = app.security.encode_id( item.id ) encoded_name = item.name - elif isinstance (item, dict): + elif isinstance(item, dict): encoded_id = item.get('id') encoded_name = item.get('name') else: @@ -2469,13 +2466,13 @@ class LibraryDatasetToolParameter( ToolParameter ): value = [value] lst = [] for item in value: - if isinstance (item, app.model.LibraryDatasetDatasetAssociation): + if isinstance(item, app.model.LibraryDatasetDatasetAssociation): lst.append(item) else: encoded_id = None - if isinstance (item, dict): + if isinstance(item, dict): encoded_id = item.get('id') - elif isinstance (item, basestring): + elif isinstance(item, basestring): encoded_id = item else: lst = [] diff --git a/lib/galaxy/tools/parameters/dataset_matcher.py b/lib/galaxy/tools/parameters/dataset_matcher.py index 7a15e28824e..d879f5cd59c 100644 --- a/lib/galaxy/tools/parameters/dataset_matcher.py +++ b/lib/galaxy/tools/parameters/dataset_matcher.py @@ -35,7 +35,7 @@ class DatasetMatcher( object ): accessible to user. """ dataset = hda.dataset - state_valid = not dataset.state in INVALID_STATES + state_valid = dataset.state not in INVALID_STATES return state_valid and ( not check_security or self.__can_access_dataset( dataset ) ) def valid_hda_match( self, hda, check_implicit_conversions=True, check_security=False ): diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index b6f1b1b6974..822be16a268 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -3,14 +3,17 @@ Support for generating the options for a SelectToolParameter dynamically (based on the values of other parameters or other aspects of the current state) """ -import operator, sys, os, logging -import basic, validation +import logging +import os +import basic +import validation from galaxy.util import string_as_bool from galaxy.model import User import galaxy.tools log = logging.getLogger(__name__) + class Filter( object ): """ A filter takes the current options list and modifies it. @@ -21,16 +24,20 @@ class Filter( object ): type = elem.get( 'type', None ) assert type is not None, "Required 'type' attribute missing from filter" return filter_types[type.strip()]( d_option, elem ) + def __init__( self, d_option, elem ): self.dynamic_option = d_option self.elem = elem + def get_dependency_name( self ): """Returns the name of any depedencies, otherwise None""" return None + def filter_options( self, options, trans, other_values ): """Returns a list of options after the filter is applied""" raise TypeError( "Abstract Method" ) + class StaticValueFilter( Filter ): """ Filters a list of options on a column by a static value. @@ -52,6 +59,7 @@ class StaticValueFilter( Filter ): assert column is not None, "Required 'column' attribute missing from filter, when loading from file" self.column = d_option.column_spec_to_index( column ) self.keep = string_as_bool( elem.get( "keep", 'True' ) ) + def filter_options( self, options, trans, other_values ): rval = [] filter_value = self.value @@ -64,6 +72,7 @@ class StaticValueFilter( Filter ): rval.append( fields ) return rval + class DataMetaFilter( Filter ): """ Filters a list of options on a column by a dataset metadata value. @@ -99,8 +108,10 @@ class DataMetaFilter( Filter ): self.column = d_option.column_spec_to_index( self.column ) self.multiple = string_as_bool( elem.get( "multiple", "False" ) ) self.separator = elem.get( "separator", "," ) + def get_dependency_name( self ): return self.ref_name + def filter_options( self, options, trans, other_values ): def compare_meta_value( file_value, dataset_value ): if isinstance( dataset_value, list ): @@ -116,10 +127,10 @@ class DataMetaFilter( Filter ): return file_value == dataset_value assert self.ref_name in other_values or ( trans is not None and trans.workflow_building_mode), "Required dependency '%s' not found in incoming values" % self.ref_name ref = other_values.get( self.ref_name, None ) - if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ) and not ( isinstance( ref, galaxy.tools.wrappers.DatasetFilenameWrapper ) ): - return [] #not a valid dataset + if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ) and not isinstance( ref, galaxy.tools.wrappers.DatasetFilenameWrapper ): + return [] # not a valid dataset meta_value = ref.metadata.get( self.key, None ) - if meta_value is None: #assert meta_value is not None, "Required metadata value '%s' not found in referenced dataset" % self.key + if meta_value is None: # assert meta_value is not None, "Required metadata value '%s' not found in referenced dataset" % self.key return [ ( disp_name, basic.UnvalidatedValue( optval ), selected ) for disp_name, optval, selected in options ] if self.column is not None: @@ -142,6 +153,7 @@ class DataMetaFilter( Filter ): options.append( ( value, value, False ) ) return options + class ParamValueFilter( Filter ): """ Filters a list of options on a column by the value of another input. @@ -173,15 +185,18 @@ class ParamValueFilter( Filter ): self.ref_attribute = self.ref_attribute.split( '.' ) else: self.ref_attribute = [] + def get_dependency_name( self ): return self.ref_name + def filter_options( self, options, trans, other_values ): - if trans is not None and trans.workflow_building_mode: return [] + if trans is not None and trans.workflow_building_mode: + return [] assert self.ref_name in other_values, "Required dependency '%s' not found in incoming values" % self.ref_name ref = other_values.get( self.ref_name, None ) for ref_attribute in self.ref_attribute: if not hasattr( ref, ref_attribute ): - return [] #ref does not have attribute, so we cannot filter, return empty list + return [] # ref does not have attribute, so we cannot filter, return empty list ref = getattr( ref, ref_attribute ) ref = str( ref ) rval = [] @@ -190,6 +205,7 @@ class ParamValueFilter( Filter ): rval.append( fields ) return rval + class UniqueValueFilter( Filter ): """ Filters a list of options to be unique by a column value. @@ -204,8 +220,10 @@ class UniqueValueFilter( Filter ): column = elem.get( "column", None ) assert column is not None, "Required 'column' attribute missing from filter" self.column = d_option.column_spec_to_index( column ) + def get_dependency_name( self ): return self.dynamic_option.dataset_ref_name + def filter_options( self, options, trans, other_values ): rval = [] skip_list = [] @@ -215,6 +233,7 @@ class UniqueValueFilter( Filter ): skip_list.append( fields[self.column] ) return rval + class MultipleSplitterFilter( Filter ): """ Turns a single line of options into multiple lines, by splitting a column and creating a line for each item. @@ -232,14 +251,16 @@ class MultipleSplitterFilter( Filter ): columns = elem.get( "column", None ) assert columns is not None, "Required 'columns' attribute missing from filter" self.columns = [ d_option.column_spec_to_index( column ) for column in columns.split( "," ) ] + def filter_options( self, options, trans, other_values ): rval = [] for fields in options: for column in self.columns: for field in fields[column].split( self.separator ): - rval.append( fields[0:column] + [field] + fields[column+1:] ) + rval.append( fields[0:column] + [field] + fields[column + 1:] ) return rval + class AttributeValueSplitterFilter( Filter ): """ Filters a list of attribute-value pairs to be unique attribute names. @@ -258,7 +279,8 @@ class AttributeValueSplitterFilter( Filter ): self.name_val_separator = elem.get( "name_val_separator", None ) self.columns = elem.get( "column", None ) assert self.columns is not None, "Required 'columns' attribute missing from filter" - self.columns = [ int ( column ) for column in self.columns.split( "," ) ] + self.columns = [ int( column ) for column in self.columns.split( "," ) ] + def filter_options( self, options, trans, other_values ): attr_names = [] rval = [] @@ -267,7 +289,7 @@ class AttributeValueSplitterFilter( Filter ): for pair in fields[column].split( self.pair_separator ): ary = pair.split( self.name_val_separator ) if len( ary ) == 2: - name, value = ary + name = ary[0] if name not in attr_names: rval.append( fields[0:column] + [name] + fields[column:] ) attr_names.append( name ) @@ -296,10 +318,11 @@ class AdditionalValueFilter( Filter ): self.index = elem.get( "index", None ) if self.index is not None: self.index = int( self.index ) + def filter_options( self, options, trans, other_values ): rval = list( options ) add_value = [] - for i in range( self.dynamic_option.largest_index + 1 ): + for _ in range( self.dynamic_option.largest_index + 1 ): add_value.append( "" ) value_col = self.dynamic_option.columns.get( 'value', 0 ) name_col = self.dynamic_option.columns.get( 'name', value_col ) @@ -312,6 +335,7 @@ class AdditionalValueFilter( Filter ): rval.append( add_value ) return rval + class RemoveValueFilter( Filter ): """ Removes a value from an options list. @@ -337,9 +361,12 @@ class RemoveValueFilter( Filter ): assert self.value is not None or ( ( self.ref_name is not None or self.meta_ref is not None )and self.metadata_key is not None ), ValueError( "Required 'value' or 'ref' and 'key' attributes missing from filter" ) self.multiple = string_as_bool( elem.get( "multiple", "False" ) ) self.separator = elem.get( "separator", "," ) + def filter_options( self, options, trans, other_values ): - if trans is not None and trans.workflow_building_mode: return options + if trans is not None and trans.workflow_building_mode: + return options assert self.value is not None or ( self.ref_name is not None and self.ref_name in other_values ) or (self.meta_ref is not None and self.meta_ref in other_values ) or ( trans is not None and trans.workflow_building_mode), Exception( "Required dependency '%s' or '%s' not found in incoming values" % ( self.ref_name, self.meta_ref ) ) + def compare_value( option_value, filter_value ): if isinstance( filter_value, list ): if self.multiple: @@ -358,11 +385,12 @@ class RemoveValueFilter( Filter ): value = other_values.get( self.ref_name ) else: data_ref = other_values.get( self.meta_ref ) - if not isinstance( data_ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ) and not ( isinstance( data_ref, galaxy.tools.wrappers.DatasetFilenameWrapper ) ): - return options #cannot modify options + if not isinstance( data_ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ) and not isinstance( data_ref, galaxy.tools.wrappers.DatasetFilenameWrapper ): + return options # cannot modify options value = data_ref.metadata.get( self.metadata_key, None ) return [ ( disp_name, optval, selected ) for disp_name, optval, selected in options if not compare_value( optval, value ) ] + class SortByColumnFilter( Filter ): """ Sorts an options list by a column @@ -377,9 +405,10 @@ class SortByColumnFilter( Filter ): column = elem.get( "column", None ) assert column is not None, "Required 'column' attribute missing from filter" self.column = d_option.column_spec_to_index( column ) + def filter_options( self, options, trans, other_values ): rval = [] - for i, fields in enumerate( options ): + for fields in options: for j in range( 0, len( rval ) ): if fields[self.column] < rval[j][self.column]: rval.insert( j, fields ) @@ -389,20 +418,21 @@ class SortByColumnFilter( Filter ): return rval -filter_types = dict( data_meta = DataMetaFilter, - param_value = ParamValueFilter, - static_value = StaticValueFilter, - unique_value = UniqueValueFilter, - multiple_splitter = MultipleSplitterFilter, - attribute_value_splitter = AttributeValueSplitterFilter, - add_value = AdditionalValueFilter, - remove_value = RemoveValueFilter, - sort_by = SortByColumnFilter ) +filter_types = dict( data_meta=DataMetaFilter, + param_value=ParamValueFilter, + static_value=StaticValueFilter, + unique_value=UniqueValueFilter, + multiple_splitter=MultipleSplitterFilter, + attribute_value_splitter=AttributeValueSplitterFilter, + add_value=AdditionalValueFilter, + remove_value=RemoveValueFilter, + sort_by=SortByColumnFilter ) + class DynamicOptions( object ): """Handles dynamically generated SelectToolParameter options""" def __init__( self, elem, tool_param ): - def load_from_parameter( from_parameter, transform_lines = None ): + def load_from_parameter( from_parameter, transform_lines=None ): obj = self.tool_param for field in from_parameter.split( '.' ): obj = getattr( obj, field ) @@ -513,7 +543,7 @@ class DynamicOptions( object ): except AttributeError: name = "a configuration file" # Perhaps this should be an error, but even a warning is useful. - log.warn( "Inconsistent number of fields (%i vs %i) in %s using separator %r, check line: %r" % \ + log.warn( "Inconsistent number of fields (%i vs %i) in %s using separator %r, check line: %r" % ( field_count, len( fields ), name, self.separator, line ) ) rval.append( fields ) return rval @@ -536,10 +566,10 @@ class DynamicOptions( object ): if self.dataset_ref_name: dataset = other_values.get( self.dataset_ref_name, None ) assert dataset is not None, "Required dataset '%s' missing from input" % self.dataset_ref_name - if not dataset: return [] #no valid dataset in history + if not dataset: + return [] # no valid dataset in history # Ensure parsing dynamic options does not consume more than a megabyte worth memory. path = dataset.file_name - file_size = os.path.getsize( path ) if os.path.getsize( path ) < 1048576: options = self.parse_file_fields( open( path ) ) else: diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index 000b4d72eeb..e994598d8b4 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -17,6 +17,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext from galaxy.model.item_attrs import Dictifiable + class Group( object, Dictifiable ): dict_collection_visible_keys = ( 'name', 'type' ) @@ -41,6 +42,7 @@ class Group( object, Dictifiable ): into the preferred value form. """ return value + def get_initial_value( self, trans, context, history=None ): """ Return the initial state/value for this group @@ -52,11 +54,12 @@ class Group( object, Dictifiable ): group_dict = super( Group, self ).to_dict( view=view, value_mapper=value_mapper ) return group_dict + class Repeat( Group ): dict_collection_visible_keys = ( 'name', 'type', 'title', 'help', 'default', 'min', 'max' ) - type = "repeat" + def __init__( self ): Group.__init__( self ) self.title = None @@ -65,11 +68,14 @@ class Repeat( Group ): self.default = 0 self.min = None self.max = None + @property def title_plural( self ): return inflector.pluralize( self.title ) + def label( self ): return "Repeat (%s)" % self.title + def value_to_basic( self, value, app ): rval = [] for d in value: @@ -81,6 +87,7 @@ class Repeat( Group ): rval_dict[ input.name ] = input.value_to_basic( d[input.name], app ) rval.append( rval_dict ) return rval + def value_from_basic( self, value, app, ignore_errors=False ): rval = [] try: @@ -103,14 +110,16 @@ class Repeat( Group ): if not ignore_errors: raise e return rval + def visit_inputs( self, prefix, value, callback ): for i, d in enumerate( value ): for input in self.inputs.itervalues(): new_prefix = prefix + "%s_%d|" % ( self.name, i ) if isinstance( input, ToolParameter ): - callback( new_prefix, input, d[input.name], parent = d ) + callback( new_prefix, input, d[input.name], parent=d ) else: input.visit_inputs( new_prefix, d[input.name], callback ) + def get_initial_value( self, trans, context, history=None ): rval = [] for i in range( self.default ): @@ -129,27 +138,32 @@ class Repeat( Group ): repeat_dict[ "inputs" ] = map( input_to_dict, self.inputs.values() ) return repeat_dict + class Section( Group ): dict_collection_visible_keys = ( 'name', 'type', 'title', 'help', 'expanded') - type = "section" + def __init__( self ): Group.__init__( self ) self.title = None self.inputs = None self.help = None self.expanded = False + @property def title_plural( self ): return inflector.pluralize( self.title ) + def label( self ): return "Section (%s)" % self.title + def value_to_basic( self, value, app ): rval = {} for input in self.inputs.itervalues(): rval[ input.name ] = input.value_to_basic( value[input.name], app ) return rval + def value_from_basic( self, value, app, ignore_errors=False ): rval = {} try: @@ -160,64 +174,79 @@ class Section( Group ): if not ignore_errors: raise e return rval + def visit_inputs( self, prefix, value, callback ): for input in self.inputs.itervalues(): if isinstance( input, ToolParameter ): - callback( prefix, input, value[input.name], parent = value ) + callback( prefix, input, value[input.name], parent=value ) else: input.visit_inputs( prefix, value[input.name], callback ) + def get_initial_value( self, trans, context, history=None ): rval = {} child_context = ExpressionContext( rval, context ) for child_input in self.inputs.itervalues(): rval[ child_input.name ] = child_input.get_initial_value( trans, child_context, history=history ) return rval + def to_dict( self, trans, view='collection', value_mapper=None ): section_dict = super( Section, self ).to_dict( trans, view=view, value_mapper=value_mapper ) + def input_to_dict( input ): return input.to_dict( trans, view=view, value_mapper=value_mapper ) + section_dict[ "inputs" ] = map( input_to_dict, self.inputs.values() ) return section_dict + class UploadDataset( Group ): type = "upload_dataset" + def __init__( self ): Group.__init__( self ) self.title = None self.inputs = None self.file_type_name = 'file_type' self.default_file_type = 'txt' - self.file_type_to_ext = { 'auto':self.default_file_type } + self.file_type_to_ext = { 'auto': self.default_file_type } self.metadata_ref = 'files_metadata' - def get_composite_dataset_name( self, context ): - #FIXME: HACK - #Special case of using 'base_name' metadata for use as Dataset name needs to be done in a General Fashion, as defined within a particular Datatype. - #We get two different types of contexts here, one straight from submitted parameters, the other after being parsed into tool inputs + def get_composite_dataset_name( self, context ): + # FIXME: HACK + # Special case of using 'base_name' metadata for use as Dataset name needs to be done in a General Fashion, as defined within a particular Datatype. + + # We get two different types of contexts here, one straight from submitted parameters, the other after being parsed into tool inputs dataset_name = context.get('files_metadata|base_name', None ) if dataset_name is None: dataset_name = context.get('files_metadata', {} ).get( 'base_name', None ) if dataset_name is None: dataset_name = 'Uploaded Composite Dataset (%s)' % self.get_file_type( context ) return dataset_name + def get_file_base_name( self, context ): - fd = context.get('files_metadata|base_name','Galaxy_Composite_file') + fd = context.get('files_metadata|base_name', 'Galaxy_Composite_file') return fd + def get_file_type( self, context ): return context.get( self.file_type_name, self.default_file_type ) + def get_datatype_ext( self, trans, context ): ext = self.get_file_type( context ) if ext in self.file_type_to_ext: - ext = self.file_type_to_ext[ext] #when using autodetect, we will use composite info from 'text', i.e. only the main file + ext = self.file_type_to_ext[ext] # when using autodetect, we will use composite info from 'text', i.e. only the main file return ext + def get_datatype( self, trans, context ): ext = self.get_datatype_ext( trans, context ) return trans.app.datatypes_registry.get_datatype_by_extension( ext ) + @property def title_plural( self ): return inflector.pluralize(self.title) + def group_title( self, context ): return "%s (%s)" % ( self.title, context.get( self.file_type_name, self.default_file_type ) ) + def title_by_index( self, trans, index, context ): d_type = self.get_datatype( trans, context ) for i, ( composite_name, composite_file ) in enumerate( d_type.writable_files.iteritems() ): @@ -229,6 +258,7 @@ class UploadDataset( Group ): rval = "%s [optional]" % rval return rval return None + def value_to_basic( self, value, app ): rval = [] for d in value: @@ -240,6 +270,7 @@ class UploadDataset( Group ): rval_dict[ input.name ] = input.value_to_basic( d[input.name], app ) rval.append( rval_dict ) return rval + def value_from_basic( self, value, app, ignore_errors=False ): rval = [] for i, d in enumerate( value ): @@ -249,34 +280,38 @@ class UploadDataset( Group ): rval_dict['__index__'] = d.get( '__index__', i ) # Restore child inputs for input in self.inputs.itervalues(): - if ignore_errors and input.name not in d: #this wasn't tested + if ignore_errors and input.name not in d: # this wasn't tested rval_dict[ input.name ] = input.get_initial_value( None, d ) else: rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors ) rval.append( rval_dict ) return rval + def visit_inputs( self, prefix, value, callback ): for i, d in enumerate( value ): for input in self.inputs.itervalues(): new_prefix = prefix + "%s_%d|" % ( self.name, i ) if isinstance( input, ToolParameter ): - callback( new_prefix, input, d[input.name], parent = d ) + callback( new_prefix, input, d[input.name], parent=d ) else: input.visit_inputs( new_prefix, d[input.name], callback ) + def get_initial_value( self, trans, context, history=None ): d_type = self.get_datatype( trans, context ) rval = [] for i, ( composite_name, composite_file ) in enumerate( d_type.writable_files.iteritems() ): rval_dict = {} - rval_dict['__index__'] = i # create __index__ + rval_dict['__index__'] = i # create __index__ for input in self.inputs.itervalues(): - rval_dict[ input.name ] = input.get_initial_value( trans, context, history=history ) #input.value_to_basic( d[input.name], app ) + rval_dict[ input.name ] = input.get_initial_value( trans, context, history=history ) # input.value_to_basic( d[input.name], app ) rval.append( rval_dict ) return rval - def get_uploaded_datasets( self, trans, context, override_name = None, override_info = None ): - def get_data_file_filename( data_file, override_name = None, override_info = None ): + + def get_uploaded_datasets( self, trans, context, override_name=None, override_info=None ): + def get_data_file_filename( data_file, override_name=None, override_info=None ): dataset_name = override_name dataset_info = override_info + def get_file_name( file_name ): file_name = file_name.split( '\\' )[-1] file_name = file_name.split( '/' )[-1] @@ -288,13 +323,13 @@ class UploadDataset( Group ): if not dataset_info: dataset_info = 'uploaded file' return Bunch( type='file', path=data_file['local_filename'], name=dataset_name ) - #return 'file', data_file['local_filename'], get_file_name( data_file.filename ), dataset_name, dataset_info + # return 'file', data_file['local_filename'], get_file_name( data_file.filename ), dataset_name, dataset_info except: # The uploaded file should've been persisted by the upload tool action return Bunch( type=None, path=None, name=None ) - #return None, None, None, None, None - def get_url_paste_urls_or_filename( group_incoming, override_name = None, override_info = None ): - filenames = [] + # return None, None, None, None, None + + def get_url_paste_urls_or_filename( group_incoming, override_name=None, override_info=None ): url_paste_file = group_incoming.get( 'url_paste', None ) if url_paste_file is not None: url_paste = open( url_paste_file, 'r' ).read( 1024 ) @@ -304,7 +339,7 @@ class UploadDataset( Group ): line = line.strip() if line: if not line.lower().startswith( 'http://' ) and not line.lower().startswith( 'ftp://' ) and not line.lower().startswith( 'https://' ): - continue # non-url line, ignore + continue # non-url line, ignore dataset_name = override_name if not dataset_name: dataset_name = line @@ -312,15 +347,16 @@ class UploadDataset( Group ): if not dataset_info: dataset_info = 'uploaded url' yield Bunch( type='url', path=line, name=dataset_name ) - #yield ( 'url', line, precreated_name, dataset_name, dataset_info ) + # yield ( 'url', line, precreated_name, dataset_name, dataset_info ) else: - dataset_name = dataset_info = precreated_name = 'Pasted Entry' #we need to differentiate between various url pastes here + dataset_name = dataset_info = precreated_name = 'Pasted Entry' # we need to differentiate between various url pastes here if override_name: dataset_name = override_name if override_info: dataset_info = override_info yield Bunch( type='file', path=url_paste_file, name=precreated_name ) - #yield ( 'file', url_paste_file, precreated_name, dataset_name, dataset_info ) + # yield ( 'file', url_paste_file, precreated_name, dataset_name, dataset_info ) + def get_one_filename( context ): data_file = context['file_data'] url_paste = context['url_paste'] @@ -335,19 +371,19 @@ class UploadDataset( Group ): space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True - file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info ) + file_bunch = get_data_file_filename( data_file, override_name=name, override_info=info ) if file_bunch.path: if url_paste is not None and url_paste.strip(): warnings.append( "All file contents specified in the paste box were ignored." ) if ftp_files: warnings.append( "All FTP uploaded file selections were ignored." ) - elif url_paste is not None and url_paste.strip(): #we need to use url_paste - for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): + elif url_paste is not None and url_paste.strip(): # we need to use url_paste + for file_bunch in get_url_paste_urls_or_filename( context, override_name=name, override_info=info ): if file_bunch.path: break if file_bunch.path and ftp_files is not None: warnings.append( "All FTP uploaded file selections were ignored." ) - elif ftp_files is not None and trans.user is not None: # look for files uploaded via FTP + elif ftp_files is not None and trans.user is not None: # look for files uploaded via FTP user_ftp_dir = trans.user_ftp_dir for ( dirpath, dirnames, filenames ) in os.walk( user_ftp_dir ): for filename in filenames: @@ -357,7 +393,7 @@ class UploadDataset( Group ): if not os.path.islink( os.path.join( dirpath, filename ) ): ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, path ) ), 'filename' : os.path.basename( path ) } - file_bunch = get_data_file_filename( ftp_data_file, override_name = name, override_info = info ) + file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info ) if file_bunch.path: break if file_bunch.path: @@ -368,10 +404,10 @@ class UploadDataset( Group ): file_bunch.space_to_tab = space_to_tab file_bunch.uuid = uuid return file_bunch, warnings + def get_filenames( context ): rval = [] data_file = context['file_data'] - url_paste = context['url_paste'] ftp_files = context['ftp_files'] uuid = context.get( 'uuid', None ) or None # Turn '' to None name = context.get( 'NAME', None ) @@ -382,14 +418,13 @@ class UploadDataset( Group ): space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True - warnings = [] - file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info ) + file_bunch = get_data_file_filename( data_file, override_name=name, override_info=info ) file_bunch.uuid = uuid if file_bunch.path: file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) - for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): + for file_bunch in get_url_paste_urls_or_filename( context, override_name=name, override_info=info ): if file_bunch.path: file_bunch.uuid = uuid file_bunch.to_posix_lines = to_posix_lines @@ -426,7 +461,7 @@ class UploadDataset( Group ): # TODO: warning to the user (could happen if file is already imported) ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, ftp_file ) ), 'filename' : os.path.basename( ftp_file ) } - file_bunch = get_data_file_filename( ftp_data_file, override_name = name, override_info = info ) + file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info ) if file_bunch.path: file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab @@ -437,13 +472,13 @@ class UploadDataset( Group ): dbkey = context.get( 'dbkey', None ) writable_files = d_type.writable_files writable_files_offset = 0 - groups_incoming = [ None for filename in writable_files ] + groups_incoming = [ None for _ in writable_files ] for group_incoming in context.get( self.name, [] ): i = int( group_incoming['__index__'] ) groups_incoming[ i ] = group_incoming if d_type.composite_type is not None: - #handle uploading of composite datatypes - #Only one Dataset can be created + # handle uploading of composite datatypes + # Only one Dataset can be created dataset = Bunch() dataset.type = 'composite' dataset.file_type = file_type @@ -453,7 +488,7 @@ class UploadDataset( Group ): dataset.metadata = {} dataset.composite_files = {} dataset.uuid = None - #load metadata + # load metadata files_metadata = context.get( self.metadata_ref, {} ) metadata_name_substition_default_dict = dict( [ ( composite_file.substitute_name_with_metadata, d_type.metadata_spec[ composite_file.substitute_name_with_metadata ].default ) for composite_file in d_type.composite_files.values() if composite_file.substitute_name_with_metadata ] ) for meta_name, meta_spec in d_type.metadata_spec.iteritems(): @@ -461,11 +496,11 @@ class UploadDataset( Group ): if meta_name in files_metadata: meta_value = files_metadata[ meta_name ] if meta_name in metadata_name_substition_default_dict: - meta_value = sanitize_for_filename( meta_value, default = metadata_name_substition_default_dict[ meta_name ] ) + meta_value = sanitize_for_filename( meta_value, default=metadata_name_substition_default_dict[ meta_name ] ) dataset.metadata[ meta_name ] = meta_value dataset.precreated_name = dataset.name = self.get_composite_dataset_name( context ) if dataset.datatype.composite_type == 'auto_primary_file': - #replace sniff here with just creating an empty file + # replace sniff here with just creating an empty file temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' ) dataset.primary_file = temp_name dataset.to_posix_lines = True @@ -477,7 +512,7 @@ class UploadDataset( Group ): dataset.to_posix_lines = file_bunch.to_posix_lines dataset.space_to_tab = file_bunch.space_to_tab dataset.warnings.extend( warnings ) - if dataset.primary_file is None:#remove this before finish, this should create an empty dataset + if dataset.primary_file is None: # remove this before finish, this should create an empty dataset raise Exception( 'No primary dataset file was available for composite upload' ) keys = [ value.name for value in writable_files.values() ] for i, group_incoming in enumerate( groups_incoming[ writable_files_offset : ] ): @@ -506,17 +541,21 @@ class UploadDataset( Group ): rval.append( dataset ) return rval + class Conditional( Group ): type = "conditional" + def __init__( self ): Group.__init__( self ) self.test_param = None self.cases = [] self.value_ref = None - self.value_ref_in_group = True #When our test_param is not part of the conditional Group, this is False + self.value_ref_in_group = True # When our test_param is not part of the conditional Group, this is False + @property def label( self ): return "Conditional (%s)" % self.name + def get_current_case( self, value, trans ): # Convert value to user representation if isinstance( value, bool ): @@ -528,6 +567,7 @@ class Conditional( Group ): if str_value == case.value: return index raise ValueError( "No case matched value:", self.name, str_value ) + def value_to_basic( self, value, app ): rval = dict() current_case = rval['__current_case__'] = value['__current_case__'] @@ -535,6 +575,7 @@ class Conditional( Group ): for input in self.cases[current_case].inputs.itervalues(): rval[ input.name ] = input.value_to_basic( value[ input.name ], app ) return rval + def value_from_basic( self, value, app, ignore_errors=False ): rval = dict() try: @@ -560,14 +601,15 @@ class Conditional( Group ): if not ignore_errors: raise e return rval + def visit_inputs( self, prefix, value, callback ): current_case = value['__current_case__'] - new_prefix = prefix + "%s|" % ( self.name ) for input in self.cases[current_case].inputs.itervalues(): if isinstance( input, ToolParameter ): - callback( prefix, input, value[input.name], parent = value ) + callback( prefix, input, value[input.name], parent=value ) else: input.visit_inputs( prefix, value[input.name], callback ) + def get_initial_value( self, trans, context, history=None ): # State for a conditional is a plain dictionary. rval = {} diff --git a/lib/galaxy/tools/parameters/input_translation.py b/lib/galaxy/tools/parameters/input_translation.py index 9f7d464be95..5e31cfafc37 100644 --- a/lib/galaxy/tools/parameters/input_translation.py +++ b/lib/galaxy/tools/parameters/input_translation.py @@ -7,6 +7,7 @@ from galaxy.util.bunch import Bunch log = logging.getLogger( __name__ ) + class ToolInputTranslator( object ): """ Handles Tool input translation. @@ -53,7 +54,7 @@ class ToolInputTranslator( object ): rval = ToolInputTranslator() for req_param in elem.findall( "request_param" ): # req_param tags must look like - #trans_list = [] + # trans_list = [] remote_name = req_param.get( "remote_name" ) galaxy_name = req_param.get( "galaxy_name" ) missing = req_param.get( "missing" ) @@ -79,9 +80,9 @@ class ToolInputTranslator( object ): value_missing = value_elem.get( 'missing' ) if None not in [ value_name, value_missing ]: append_dict[ value_name ] = value_missing - append_param = Bunch( separator = separator, first_separator = first_separator, join_str = join_str, append_dict = append_dict ) + append_param = Bunch( separator=separator, first_separator=first_separator, join_str=join_str, append_dict=append_dict ) - rval.param_trans_dict[ remote_name ] = Bunch( galaxy_name = galaxy_name, missing = missing, value_trans = value_trans, append_param = append_param ) + rval.param_trans_dict[ remote_name ] = Bunch( galaxy_name=galaxy_name, missing=missing, value_trans=value_trans, append_param=append_param ) return rval @@ -93,8 +94,8 @@ class ToolInputTranslator( object ): update params in-place """ for remote_name, translator in self.param_trans_dict.iteritems(): - galaxy_name = translator.galaxy_name #NB: if a param by name galaxy_name is provided, it is always thrown away unless galaxy_name == remote_name - value = params.get( remote_name, translator.missing ) #get value from input params, or use default value specified in tool config + galaxy_name = translator.galaxy_name # NB: if a param by name galaxy_name is provided, it is always thrown away unless galaxy_name == remote_name + value = params.get( remote_name, translator.missing ) # get value from input params, or use default value specified in tool config if translator.value_trans and value in translator.value_trans: value = translator.value_trans[ value ] if translator.append_param: diff --git a/lib/galaxy/tools/parameters/output_collect.py b/lib/galaxy/tools/parameters/output_collect.py index f886d62883b..12fa418c46c 100644 --- a/lib/galaxy/tools/parameters/output_collect.py +++ b/lib/galaxy/tools/parameters/output_collect.py @@ -229,7 +229,7 @@ def collect_primary_datasets( tool, output, job_working_directory, input_ext ): app.object_store.update_from_file( outdata.dataset, file_name=filename, create=True ) primary_output_assigned = True continue - if not name in primary_datasets: + if name not in primary_datasets: primary_datasets[ name ] = {} visible = fields_match.visible ext = fields_match.ext diff --git a/lib/galaxy/tools/parameters/sanitize.py b/lib/galaxy/tools/parameters/sanitize.py index 921985aa96f..5b70f5742d6 100644 --- a/lib/galaxy/tools/parameters/sanitize.py +++ b/lib/galaxy/tools/parameters/sanitize.py @@ -8,6 +8,7 @@ import galaxy.util log = logging.getLogger( __name__ ) + class ToolParameterSanitizer( object ): """ Handles tool parameter specific sanitizing. @@ -41,15 +42,15 @@ class ToolParameterSanitizer( object ): True """ - VALID_PRESET = { 'default':( string.letters + string.digits +" -=_.()/+*^,:?!" ), 'none':'' } - MAPPING_PRESET = { 'default':galaxy.util.mapped_chars, 'none':{} } + VALID_PRESET = { 'default': ( string.letters + string.digits + " -=_.()/+*^,:?!" ), 'none': '' } + MAPPING_PRESET = { 'default': galaxy.util.mapped_chars, 'none': {} } DEFAULT_INVALID_CHAR = 'X' - #class methods + # class methods @classmethod def from_element( cls, elem ): """Loads the proper filter by the type attribute of elem""" - #TODO: Add ability to generically specify a method to use for sanitizing input via specification in tool XML + # TODO: Add ability to generically specify a method to use for sanitizing input via specification in tool XML rval = ToolParameterSanitizer() rval._invalid_char = elem.get( 'invalid_char', cls.DEFAULT_INVALID_CHAR ) rval.sanitize = galaxy.util.string_as_bool( elem.get( 'sanitize', 'True' ) ) @@ -59,11 +60,11 @@ class ToolParameterSanitizer( object ): preset = rval.get_valid_by_name( action_elem.get( 'preset', 'none' ) ) valid_value = [ val for val in action_elem.get( 'value', [] ) ] if action_elem.tag.lower() == 'add': - for val in ( preset + valid_value ): + for val in preset + valid_value: if val not in rval._valid_chars: rval._valid_chars.append( val ) elif action_elem.tag.lower() == 'remove': - for val in ( preset + valid_value ): + for val in preset + valid_value: while val in rval._valid_chars: rval._valid_chars.remove( val ) else: @@ -116,13 +117,13 @@ class ToolParameterSanitizer( object ): else: log.debug( 'Invalid preset name specified: %s' % split_name ) return rval - #end class methods + # end class methods def __init__( self ): - self._valid_chars = [] #List of valid characters - self._mapped_chars = {} #Replace a char with a any number of characters - self._invalid_char = self.DEFAULT_INVALID_CHAR #Replace invalid characters with this character - self.sanitize = True #Simply pass back the passed in value + self._valid_chars = [] # List of valid characters + self._mapped_chars = {} # Replace a char with a any number of characters + self._invalid_char = self.DEFAULT_INVALID_CHAR # Replace invalid characters with this character + self.sanitize = True # Simply pass back the passed in value def restore_text( self, text ): """Restores sanitized text""" @@ -138,7 +139,7 @@ class ToolParameterSanitizer( object ): elif isinstance( value, list ): return map( self.restore_text, value ) else: - raise Exception, 'Unknown parameter type (%s:%s)' % ( type( value ), value ) + raise Exception('Unknown parameter type (%s:%s)' % ( type( value ), value )) return value def sanitize_text( self, text ): @@ -164,4 +165,4 @@ class ToolParameterSanitizer( object ): elif isinstance( value, list ): return map( self.sanitize_text, value ) else: - raise Exception, 'Unknown parameter type (%s:%s)' % ( type( value ), value ) + raise Exception('Unknown parameter type (%s:%s)' % ( type( value ), value )) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index 588949d63eb..3a29ee57dee 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -2,16 +2,18 @@ Classes related to parameter validation. """ -import os, re, logging -from xml.etree.ElementTree import XML +import logging +import re from galaxy import model log = logging.getLogger( __name__ ) + class LateValidationError( Exception ): def __init__( self, message ): self.message = message + class Validator( object ): """ A validator checks that a value meets some conditions OR raises ValueError @@ -21,14 +23,17 @@ class Validator( object ): type = elem.get( 'type', None ) assert type is not None, "Required 'type' attribute missing from validator" return validator_types[type].from_element( param, elem ) + def validate( self, value, history=None ): raise TypeError( "Abstract Method" ) + class RegexValidator( Validator ): """ Validator that evaluates a regular expression - >>> from galaxy.tools.parameters import ToolParameter + >>> from xml.etree.ElementTree import XML + >>> from galaxy.tools.parameters.basic import ToolParameter >>> p = ToolParameter.build( None, XML( ''' ... ... [Ff]oo @@ -44,20 +49,24 @@ class RegexValidator( Validator ): @classmethod def from_element( cls, param, elem ): return cls( elem.get( 'message' ), elem.text ) + def __init__( self, message, expression ): self.message = message # Compile later. RE objects used to not be thread safe. Not sure about # the sre module. self.expression = expression + def validate( self, value, history=None ): if re.match( self.expression, value ) is None: raise ValueError( self.message ) + class ExpressionValidator( Validator ): """ Validator that evaluates a python expression using the value - >>> from galaxy.tools.parameters import ToolParameter + >>> from xml.etree.ElementTree import XML + >>> from galaxy.tools.parameters.basic import ToolParameter >>> p = ToolParameter.build( None, XML( ''' ... ... value.lower() == "foo" @@ -73,11 +82,13 @@ class ExpressionValidator( Validator ): @classmethod def from_element( cls, param, elem ): return cls( elem.get( 'message' ), elem.text, elem.get( 'substitute_value_in_message' ) ) + def __init__( self, message, expression, substitute_value_in_message ): self.message = message self.substitute_value_in_message = substitute_value_in_message # Save compiled expression, code objects are thread safe (right?) self.expression = compile( expression, '', 'eval' ) + def validate( self, value, history=None ): if not( eval( self.expression, dict( value=value ) ) ): message = self.message @@ -85,11 +96,13 @@ class ExpressionValidator( Validator ): message = message % value raise ValueError( message ) + class InRangeValidator( Validator ): """ Validator that ensures a number is in a specific range - >>> from galaxy.tools.parameters import ToolParameter + >>> from xml.etree.ElementTree import XML + >>> from galaxy.tools.parameters.basic import ToolParameter >>> p = ToolParameter.build( None, XML( ''' ... ... @@ -106,6 +119,7 @@ class InRangeValidator( Validator ): @classmethod def from_element( cls, param, elem ): return cls( elem.get( 'message', None ), elem.get( 'min' ), elem.get( 'max' ) ) + def __init__( self, message, range_min, range_max ): self.min = float( range_min if range_min is not None else '-inf' ) self.max = float( range_max if range_max is not None else 'inf' ) @@ -114,15 +128,18 @@ class InRangeValidator( Validator ): self_min_str = str( self.min ).rstrip( '0' ).rstrip( '.' ) self_max_str = str( self.max ).rstrip( '0' ).rstrip( '.' ) self.message = message or "Value must be between %s and %s" % ( self_min_str, self_max_str ) + def validate( self, value, history=None ): if not( self.min <= float( value ) <= self.max ): raise ValueError( self.message ) + class LengthValidator( Validator ): """ Validator that ensures the length of the provided string (value) is in a specific range - >>> from galaxy.tools.parameters import ToolParameter + >>> from xml.etree.ElementTree import XML + >>> from galaxy.tools.parameters.basic import ToolParameter >>> p = ToolParameter.build( None, XML( ''' ... ... @@ -142,6 +159,7 @@ class LengthValidator( Validator ): @classmethod def from_element( cls, param, elem ): return cls( elem.get( 'message', None ), elem.get( 'min', None ), elem.get( 'max', None ) ) + def __init__( self, message, length_min, length_max ): self.message = message if length_min is not None: @@ -150,47 +168,55 @@ class LengthValidator( Validator ): length_max = int( length_max ) self.min = length_min self.max = length_max + def validate( self, value, history=None ): if self.min is not None and len( value ) < self.min: raise ValueError( self.message or ( "Must have length of at least %d" % self.min ) ) if self.max is not None and len( value ) > self.max: raise ValueError( self.message or ( "Must have length no more than %d" % self.max ) ) + class DatasetOkValidator( Validator ): """ Validator that checks if a dataset is in an 'ok' state """ def __init__( self, message=None ): self.message = message + @classmethod def from_element( cls, param, elem ): return cls( elem.get( 'message', None ) ) + def validate( self, value, history=None ): if value and value.state != model.Dataset.states.OK: if self.message is None: self.message = "The selected dataset is still being generated, select another dataset or wait until it is completed" raise ValueError( self.message ) + class MetadataValidator( Validator ): """ Validator that checks for missing metadata """ - def __init__( self, message = None, check = "", skip = "" ): + def __init__( self, message=None, check="", skip="" ): self.message = message self.check = check.split( "," ) self.skip = skip.split( "," ) + @classmethod def from_element( cls, param, elem ): return cls( message=elem.get( 'message', None ), check=elem.get( 'check', "" ), skip=elem.get( 'skip', "" ) ) + def validate( self, value, history=None ): if value: if not isinstance( value, model.DatasetInstance ): raise ValueError( 'A non-dataset value was provided.' ) - if value.missing_meta( check = self.check, skip = self.skip ): + if value.missing_meta( check=self.check, skip=self.skip ): if self.message is None: self.message = "Metadata missing, click the pencil icon in the history item to edit / save the metadata attributes" raise ValueError( self.message ) + class UnspecifiedBuildValidator( Validator ): """ Validator that checks for dbkey not equal to '?' @@ -200,11 +226,13 @@ class UnspecifiedBuildValidator( Validator ): self.message = "Unspecified genome build, click the pencil icon in the history item to set the genome build" else: self.message = message + @classmethod def from_element( cls, param, elem ): return cls( elem.get( 'message', None ) ) + def validate( self, value, history=None ): - #if value is None, we cannot validate + # if value is None, we cannot validate if value: dbkey = value.metadata.dbkey if isinstance( dbkey, list ): @@ -212,32 +240,39 @@ class UnspecifiedBuildValidator( Validator ): if dbkey == '?': raise ValueError( self.message ) + class NoOptionsValidator( Validator ): """Validator that checks for empty select list""" def __init__( self, message=None ): self.message = message + @classmethod def from_element( cls, param, elem ): return cls( elem.get( 'message', None ) ) + def validate( self, value, history=None ): if value is None: if self.message is None: self.message = "No options available for selection" raise ValueError( self.message ) + class EmptyTextfieldValidator( Validator ): """Validator that checks for empty text field""" def __init__( self, message=None ): self.message = message + @classmethod def from_element( cls, param, elem ): return cls( elem.get( 'message', None ) ) + def validate( self, value, history=None ): if value == '': if self.message is None: self.message = "Field requires a value" raise ValueError( self.message ) + class MetadataInFileColumnValidator( Validator ): """ Validator that checks if the value for a dataset's metadata item exists in a file. @@ -256,6 +291,7 @@ class MetadataInFileColumnValidator( Validator ): if line_startswith: line_startswith = line_startswith.strip() return cls( filename, metadata_name, metadata_column, message, line_startswith ) + def __init__( self, filename, metadata_name, metadata_column, message="Value for metadata not found.", line_startswith=None ): self.metadata_name = metadata_name self.message = message @@ -265,13 +301,16 @@ class MetadataInFileColumnValidator( Validator ): fields = line.split( '\t' ) if metadata_column < len( fields ): self.valid_values.append( fields[metadata_column].strip() ) - def validate( self, value, history = None ): - if not value: return + + def validate( self, value, history=None ): + if not value: + return if hasattr( value, "metadata" ): if value.metadata.spec[self.metadata_name].param.to_string( value.metadata.get( self.metadata_name ) ) in self.valid_values: return raise ValueError( self.message ) + class MetadataInDataTableColumnValidator( Validator ): """ Validator that checks if the value for a dataset's metadata item exists in a file. @@ -313,8 +352,9 @@ class MetadataInDataTableColumnValidator( Validator ): if self._metadata_column < len( fields ): self.valid_values.append( fields[ self._metadata_column ] ) - def validate( self, value, history = None ): - if not value: return + def validate( self, value, history=None ): + if not value: + return if hasattr( value, "metadata" ): if not self._tool_data_table.is_current_version( self._data_table_content_version ): log.debug( 'MetadataInDataTableColumnValidator values are out of sync with data table (%s), updating validator.', self._tool_data_table.name ) @@ -336,8 +376,9 @@ validator_types = dict( expression=ExpressionValidator, dataset_metadata_in_data_table=MetadataInDataTableColumnValidator, dataset_ok_validator=DatasetOkValidator ) + def get_suite(): """Get unittest suite for this module""" - import doctest, sys + import doctest + import sys return doctest.DocTestSuite( sys.modules[__name__] ) - diff --git a/lib/galaxy/tools/parameters/wrapped.py b/lib/galaxy/tools/parameters/wrapped.py index 5765e40cb7b..89aed607ca4 100644 --- a/lib/galaxy/tools/parameters/wrapped.py +++ b/lib/galaxy/tools/parameters/wrapped.py @@ -1,5 +1,3 @@ -import galaxy.tools - from galaxy.tools.parameters.basic import ( DataToolParameter, DataCollectionToolParameter, @@ -59,15 +57,15 @@ class WrappedParameters( object ): elif isinstance( input, DataToolParameter ) and input.multiple: input_values[ input.name ] = \ DatasetListWrapper( input_values[ input.name ], - datatypes_registry=trans.app.datatypes_registry, - tool=tool, - name=input.name ) + datatypes_registry=trans.app.datatypes_registry, + tool=tool, + name=input.name ) elif isinstance( input, DataToolParameter ): input_values[ input.name ] = \ DatasetFilenameWrapper( input_values[ input.name ], - datatypes_registry=trans.app.datatypes_registry, - tool=tool, - name=input.name ) + datatypes_registry=trans.app.datatypes_registry, + tool=tool, + name=input.name ) elif isinstance( input, SelectToolParameter ): input_values[ input.name ] = SelectToolParameterWrapper( input, input_values[ input.name ], tool.app, other_values=incoming ) elif isinstance( input, DataCollectionToolParameter ): @@ -111,4 +109,4 @@ def make_list_copy( from_list ): return new_list -__all__ = [ WrappedParameters, make_dict_copy ] +__all__ = [ 'WrappedParameters', 'make_dict_copy' ] diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 66be0ff435f..2c791cf210e 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -236,7 +236,7 @@ class XmlToolSource(ToolSource): output.tool = tool output.from_work_dir = data_elem.get("from_work_dir", None) output.hidden = string_as_bool( data_elem.get("hidden", "") ) - output.actions = galaxy.tools.ToolOutputActionGroup( output, data_elem.find( 'actions' ) ) + output.actions = galaxy.tools.parameters.output.ToolOutputActionGroup( output, data_elem.find( 'actions' ) ) output.dataset_collectors = output_collect.dataset_collectors_from_elem( data_elem ) return output diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py index 6ab094d51f3..aa19ea77327 100644 --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -87,7 +87,7 @@ class YamlToolSource(ToolSource): output.tool = tool output.from_work_dir = output_dict.get("from_work_dir", None) output.hidden = output_dict.get("hidden", "") - output.actions = galaxy.tools.ToolOutputActionGroup( output, None ) + output.actions = galaxy.tools.parameters.output.ToolOutputActionGroup( output, None ) discover_datasets_dicts = output_dict.get( "discover_datasets", [] ) if isinstance( discover_datasets_dicts, dict ): discover_datasets_dicts = [ discover_datasets_dicts ] diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index d7ee29f3c8d..3293ec09c54 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -18,6 +18,7 @@ schema = Schema( id=STORED, import logging log = logging.getLogger( __name__ ) + class ToolBoxSearch( object ): """ Support searching tools in a toolbox. This implementation uses diff --git a/lib/galaxy/tools/util/galaxyops/__init__.py b/lib/galaxy/tools/util/galaxyops/__init__.py index b8c4b36d0b0..d2316b2e893 100644 --- a/lib/galaxy/tools/util/galaxyops/__init__.py +++ b/lib/galaxy/tools/util/galaxyops/__init__.py @@ -1,7 +1,6 @@ """Utility functions for galaxyops""" import sys -from bx.bitset import * -from bx.intervals.io import * + def warn( msg ): # TODO: since everything printed to stderr results in job.state = error, we @@ -9,6 +8,7 @@ def warn( msg ): print >> sys.stderr, msg sys.exit( 1 ) + def fail( msg ): print >> sys.stderr, msg sys.exit( 1 ) @@ -16,6 +16,7 @@ def fail( msg ): # Default chrom, start, end, strand cols for a bed file BED_DEFAULT_COLS = 0, 1, 2, 5 + def parse_cols_arg( cols ): """Parse a columns command line argument into a four-tuple""" if cols: @@ -28,10 +29,12 @@ def parse_cols_arg( cols ): else: return BED_DEFAULT_COLS + def default_printer( stream, exc, obj ): print >> stream, "%d: %s" % ( obj.linenum, obj.current_line ) print >> stream, "\tError: %s" % ( str(exc) ) + def skipped( reader, filedesc="" ): first_line, line_contents, problem = reader.skipped_lines[0] return 'Skipped %d invalid lines%s, 1st line #%d: "%s", problem: %s' % ( reader.skipped, filedesc, first_line, line_contents, problem ) diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py index 09fc7cc1214..b40730f09bf 100644 --- a/lib/galaxy/tools/util/maf_utilities.py +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -2,13 +2,17 @@ """ Provides wrappers and utilities for working with MAF files and alignments. """ -#Dan Blankenberg -import pkg_resources; pkg_resources.require( "bx-python" ) +# Dan Blankenberg +import logging +import os +import string +import sys +import tempfile +import pkg_resources +pkg_resources.require( "bx-python" ) import bx.align.maf import bx.intervals import bx.interval_index_file -import sys, os, string, tempfile -import logging from errno import EMFILE import resource from copy import deepcopy @@ -21,6 +25,7 @@ log = logging.getLogger(__name__) GAP_CHARS = [ '-' ] SRC_SPLIT_CHAR = '.' + def src_split( src ): fields = src.split( SRC_SPLIT_CHAR, 1 ) spec = fields.pop( 0 ) @@ -30,11 +35,13 @@ def src_split( src ): chrom = spec return spec, chrom -def src_merge( spec, chrom, contig = None ): + +def src_merge( spec, chrom, contig=None ): if None in [ spec, chrom ]: spec = chrom = spec or chrom return bx.align.maf.src_merge( spec, chrom, contig ) + def get_species_in_block( block ): species = [] for c in block.components: @@ -43,14 +50,15 @@ def get_species_in_block( block ): species.append( spec ) return species -def tool_fail( msg = "Unknown Error" ): + +def tool_fail( msg="Unknown Error" ): print >> sys.stderr, "Fatal Error: %s" % msg sys.exit() class TempFileHandler( object ): ''' - Handles creating, opening, closing, and deleting of Temp files, with a + Handles creating, opening, closing, and deleting of Temp files, with a maximum number of files open at one time. ''' @@ -75,7 +83,7 @@ class TempFileHandler( object ): index = len( self.files ) temp_kwds = dict( self.kwds ) temp_kwds.update( kwds ) - # Being able to use delete=True here, would simplify a bit, + # Being able to use delete=True here, would simplify a bit, # but we support python2.4 in these tools while True: try: @@ -121,17 +129,17 @@ class TempFileHandler( object ): self.files[ index ].flush() def __del__( self ): - for i in xrange( len( self.files ) ): + for i in xrange( len( self.files ) ): self.close( i, delete=True ) -#an object corresponding to a reference layered alignment +# an object corresponding to a reference layered alignment class RegionAlignment( object ): DNA_COMPLEMENT = string.maketrans( "ACGTacgt", "TGCAtgca" ) - MAX_SEQUENCE_SIZE = sys.maxint #Maximum length of sequence allowed + MAX_SEQUENCE_SIZE = sys.maxint # Maximum length of sequence allowed - def __init__( self, size, species = [], temp_file_handler = None ): + def __init__( self, size, species=[], temp_file_handler=None ): assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % ( size, self.MAX_SEQUENCE_SIZE ) self.size = size if not temp_file_handler: @@ -143,49 +151,57 @@ class RegionAlignment( object ): for spec in species: self.add_species( spec ) - #add a species to the alignment + # add a species to the alignment def add_species( self, species ): - #make temporary sequence files + # make temporary sequence files file_index, fh = self.temp_file_handler.get_open_tempfile() self.sequences[species] = file_index fh.write( "-" * self.size ) - #returns the names for species found in alignment, skipping names as requested - def get_species_names( self, skip = [] ): - if not isinstance( skip, list ): skip = [skip] + # returns the names for species found in alignment, skipping names as requested + def get_species_names( self, skip=[] ): + if not isinstance( skip, list ): + skip = [skip] names = self.sequences.keys() for name in skip: - try: names.remove( name ) - except: pass + try: + names.remove( name ) + except: + pass return names - #returns the sequence for a species + # returns the sequence for a species def get_sequence( self, species ): file_index, fh = self.temp_file_handler.get_open_tempfile( self.sequences[species] ) fh.seek( 0 ) return fh.read() - #returns the reverse complement of the sequence for a species + # returns the reverse complement of the sequence for a species def get_sequence_reverse_complement( self, species ): complement = [base for base in self.get_sequence( species ).translate( self.DNA_COMPLEMENT )] complement.reverse() return "".join( complement ) - #sets a position for a species + # sets a position for a species def set_position( self, index, species, base ): - if len( base ) != 1: raise Exception( "A genomic position can only have a length of 1." ) + if len( base ) != 1: + raise Exception( "A genomic position can only have a length of 1." ) return self.set_range( index, species, base ) - #sets a range for a species + # sets a range for a species + def set_range( self, index, species, bases ): - if index >= self.size or index < 0: raise Exception( "Your index (%i) is out of range (0 - %i)." % ( index, self.size - 1 ) ) - if len( bases ) == 0: raise Exception( "A set of genomic positions can only have a positive length." ) - if species not in self.sequences.keys(): self.add_species( species ) + if index >= self.size or index < 0: + raise Exception( "Your index (%i) is out of range (0 - %i)." % ( index, self.size - 1 ) ) + if len( bases ) == 0: + raise Exception( "A set of genomic positions can only have a positive length." ) + if species not in self.sequences.keys(): + self.add_species( species ) file_index, fh = self.temp_file_handler.get_open_tempfile( self.sequences[species] ) fh.seek( index ) fh.write( bases ) - #Flush temp file of specified species, or all species - def flush( self, species = None ): + # Flush temp file of specified species, or all species + def flush( self, species=None ): if species is None: species = self.sequences.keys() elif not isinstance( species, list ): @@ -193,18 +209,20 @@ class RegionAlignment( object ): for spec in species: self.temp_file_handler.flush( self.sequences[spec] ) + class GenomicRegionAlignment( RegionAlignment ): - def __init__( self, start, end, species = [], temp_file_handler = None ): + def __init__( self, start, end, species=[], temp_file_handler=None ): RegionAlignment.__init__( self, end - start, species, temp_file_handler=temp_file_handler ) self.start = start self.end = end + class SplicedAlignment( object ): DNA_COMPLEMENT = string.maketrans( "ACGTacgt", "TGCAtgca" ) - def __init__( self, exon_starts, exon_ends, species = [], temp_file_handler = None ): + def __init__( self, exon_starts, exon_ends, species=[], temp_file_handler=None ): if not isinstance( exon_starts, list ): exon_starts = [exon_starts] if not isinstance( exon_ends, list ): @@ -217,17 +235,18 @@ class SplicedAlignment( object ): for i in range( len( exon_starts ) ): self.exons.append( GenomicRegionAlignment( exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler ) ) - #returns the names for species found in alignment, skipping names as requested - def get_species_names( self, skip = [] ): - if not isinstance( skip, list ): skip = [skip] + # returns the names for species found in alignment, skipping names as requested + def get_species_names( self, skip=[] ): + if not isinstance( skip, list ): + skip = [skip] names = [] for exon in self.exons: - for name in exon.get_species_names( skip = skip ): + for name in exon.get_species_names( skip=skip ): if name not in names: names.append( name ) return names - #returns the sequence for a species + # returns the sequence for a species def get_sequence( self, species ): index, fh = self.temp_file_handler.get_open_tempfile() for exon in self.exons: @@ -244,45 +263,50 @@ class SplicedAlignment( object ): self.temp_file_handler.close( index, delete=True ) return rval - #returns the reverse complement of the sequence for a species + # returns the reverse complement of the sequence for a species def get_sequence_reverse_complement( self, species ): complement = [base for base in self.get_sequence( species ).translate( self.DNA_COMPLEMENT )] complement.reverse() return "".join( complement ) - #Start and end of coding region + # Start and end of coding region @property def start( self ): return self.exons[0].start + @property def end( self ): return self.exons[-1].end -#Open a MAF index using a UID + +# Open a MAF index using a UID def maf_index_by_uid( maf_uid, index_location_file ): for line in open( index_location_file ): try: - #read each line, if not enough fields, go to next line - if line[0:1] == "#" : continue + # read each line, if not enough fields, go to next line + if line[0:1] == "#": + continue fields = line.split('\t') if maf_uid == fields[1]: try: maf_files = fields[4].replace( "\n", "" ).replace( "\r", "" ).split( "," ) - return bx.align.maf.MultiIndexed( maf_files, keep_open = True, parse_e_rows = False ) + return bx.align.maf.MultiIndexed( maf_files, keep_open=True, parse_e_rows=False ) except Exception, e: raise Exception( 'MAF UID (%s) found, but configuration appears to be malformed: %s' % ( maf_uid, e ) ) except: pass return None -#return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created -def open_or_build_maf_index( maf_file, index_filename, species = None ): - try: - return ( bx.align.maf.Indexed( maf_file, index_filename = index_filename, keep_open = True, parse_e_rows = False ), None ) - except: - return build_maf_index( maf_file, species = species ) -def build_maf_index_species_chromosomes( filename, index_species = None ): +# return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created +def open_or_build_maf_index( maf_file, index_filename, species=None ): + try: + return ( bx.align.maf.Indexed( maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False ), None ) + except: + return build_maf_index( maf_file, species=species ) + + +def build_maf_index_species_chromosomes( filename, index_species=None ): species = [] species_chromosomes = {} indexes = bx.interval_index_file.Indexes() @@ -312,51 +336,55 @@ def build_maf_index_species_chromosomes( filename, index_species = None ): forward_strand_start = int( forward_strand_start ) forward_strand_end = int( forward_strand_end ) except ValueError: - continue #start and end are not integers, can't add component to index, goto next component - #this likely only occurs when parse_e_rows is True? - #could a species exist as only e rows? should the + continue # start and end are not integers, can't add component to index, goto next component + # this likely only occurs when parse_e_rows is True? + # could a species exist as only e rows? should the if forward_strand_end > forward_strand_start: - #require positive length; i.e. certain lines have start = end = 0 and cannot be indexed + # require positive length; i.e. certain lines have start = end = 0 and cannot be indexed indexes.add( c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size ) except Exception, e: - #most likely a bad MAF + # most likely a bad MAF log.debug( 'Building MAF index on %s failed: %s' % ( filename, e ) ) return ( None, [], {}, 0 ) return ( indexes, species, species_chromosomes, blocks ) -#builds and returns ( index, index_filename ) for specified maf_file -def build_maf_index( maf_file, species = None ): + +# builds and returns ( index, index_filename ) for specified maf_file +def build_maf_index( maf_file, species=None ): indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes( maf_file, species ) if indexes is not None: fd, index_filename = tempfile.mkstemp() out = os.fdopen( fd, 'w' ) indexes.write( out ) out.close() - return ( bx.align.maf.Indexed( maf_file, index_filename = index_filename, keep_open = True, parse_e_rows = False ), index_filename ) + return ( bx.align.maf.Indexed( maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False ), index_filename ) return ( None, None ) + def component_overlaps_region( c, region ): - if c is None: return False + if c is None: + return False start, end = c.get_forward_strand_start(), c.get_forward_strand_end() if region.start >= end or region.end <= start: return False return True -def chop_block_by_region( block, src, region, species = None, mincols = 0 ): + +def chop_block_by_region( block, src, region, species=None, mincols=0 ): # This chopping method was designed to maintain consistency with how start/end padding gaps have been working in Galaxy thus far: # behavior as seen when forcing blocks to be '+' relative to src sequence (ref) and using block.slice_by_component( ref, slice_start, slice_end ) # whether-or-not this is the 'correct' behavior is questionable, but this will at least maintain consistency # comments welcome - slice_start = block.text_size #max for the min() - slice_end = 0 #min for the max() - old_score = block.score #save old score for later use + slice_start = block.text_size # max for the min() + slice_end = 0 # min for the max() + old_score = block.score # save old score for later use # We no longer assume only one occurance of src per block, so we need to check them all for c in iter_components_by_src( block, src ): if component_overlaps_region( c, region ): if c.text is not None: rev_strand = False if c.strand == "-": - #We want our coord_to_col coordinates to be returned from positive stranded component + # We want our coord_to_col coordinates to be returned from positive stranded component rev_strand = True c = c.reverse_complement() start = max( region.start, c.start ) @@ -364,7 +392,7 @@ def chop_block_by_region( block, src, region, species = None, mincols = 0 ): start = c.coord_to_col( start ) end = c.coord_to_col( end ) if rev_strand: - #need to orient slice coordinates to the original block direction + # need to orient slice coordinates to the original block direction slice_len = end - start end = len( c.text ) - start start = end - slice_len @@ -382,39 +410,45 @@ def chop_block_by_region( block, src, region, species = None, mincols = 0 ): return block return None -def orient_block_by_region( block, src, region, force_strand = None ): - #loop through components matching src, - #make sure each of these components overlap region - #cache strand for each of overlaping regions - #if force_strand / region.strand not in strand cache, reverse complement - ### we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing + +def orient_block_by_region( block, src, region, force_strand=None ): + # loop through components matching src, + # make sure each of these components overlap region + # cache strand for each of overlaping regions + # if force_strand / region.strand not in strand cache, reverse complement + # we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing strands = [ c.strand for c in iter_components_by_src( block, src ) if component_overlaps_region( c, region ) ] if strands and ( force_strand is None and region.strand not in strands ) or ( force_strand is not None and force_strand not in strands ): block = block.reverse_complement() return block -def get_oriented_chopped_blocks_for_region( index, src, region, species = None, mincols = 0, force_strand = None ): + +def get_oriented_chopped_blocks_for_region( index, src, region, species=None, mincols=0, force_strand=None ): for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region( index, src, region, species, mincols, force_strand ): yield block -def get_oriented_chopped_blocks_with_index_offset_for_region( index, src, region, species = None, mincols = 0, force_strand = None ): + + +def get_oriented_chopped_blocks_with_index_offset_for_region( index, src, region, species=None, mincols=0, force_strand=None ): for block, idx, offset in get_chopped_blocks_with_index_offset_for_region( index, src, region, species, mincols ): yield orient_block_by_region( block, src, region, force_strand ), idx, offset -#split a block with multiple occurances of src into one block per src + +# split a block with multiple occurances of src into one block per src def iter_blocks_split_by_src( block, src ): for src_c in iter_components_by_src( block, src ): new_block = bx.align.Alignment( score=block.score, attributes=deepcopy( block.attributes ) ) new_block.text_size = block.text_size for c in block.components: if c == src_c or c.src != src: - new_block.add_component( deepcopy( c ) ) #components have reference to alignment, dont want to loose reference to original alignment block in original components + new_block.add_component( deepcopy( c ) ) # components have reference to alignment, dont want to loose reference to original alignment block in original components yield new_block -#split a block into multiple blocks with all combinations of a species appearing only once per block -def iter_blocks_split_by_species( block, species = None ): + +# split a block into multiple blocks with all combinations of a species appearing only once per block +def iter_blocks_split_by_species( block, species=None ): def __split_components_by_species( components_by_species, new_block ): if components_by_species: - #more species with components to add to this block + # more species with components to add to this block components_by_species = deepcopy( components_by_species ) spec_comps = components_by_species.pop( 0 ) for c in spec_comps: @@ -423,10 +457,10 @@ def iter_blocks_split_by_species( block, species = None ): for value in __split_components_by_species( components_by_species, newer_block ): yield value else: - #no more components to add, yield this block + # no more components to add, yield this block yield new_block - #divide components by species + # divide components by species spec_dict = {} if not species: species = [] @@ -442,41 +476,47 @@ def iter_blocks_split_by_species( block, species = None ): for c in iter_components_by_src_start( block, spec ): spec_dict[ spec ].append( c ) - empty_block = bx.align.Alignment( score=block.score, attributes=deepcopy( block.attributes ) ) #should we copy attributes? + empty_block = bx.align.Alignment( score=block.score, attributes=deepcopy( block.attributes ) ) # should we copy attributes? empty_block.text_size = block.text_size - #call recursive function to split into each combo of spec/blocks + # call recursive function to split into each combo of spec/blocks for value in __split_components_by_species( spec_dict.values(), empty_block ): - sort_block_components_by_block( value, block ) #restore original component order + sort_block_components_by_block( value, block ) # restore original component order yield value -#generator yielding only chopped and valid blocks for a specified region -def get_chopped_blocks_for_region( index, src, region, species = None, mincols = 0 ): +# generator yielding only chopped and valid blocks for a specified region +def get_chopped_blocks_for_region( index, src, region, species=None, mincols=0 ): for block, idx, offset in get_chopped_blocks_with_index_offset_for_region( index, src, region, species, mincols ): yield block -def get_chopped_blocks_with_index_offset_for_region( index, src, region, species = None, mincols = 0 ): + + +def get_chopped_blocks_with_index_offset_for_region( index, src, region, species=None, mincols=0 ): for block, idx, offset in index.get_as_iterator_with_index_and_offset( src, region.start, region.end ): block = chop_block_by_region( block, src, region, species, mincols ) if block is not None: yield block, idx, offset -#returns a filled region alignment for specified regions -def get_region_alignment( index, primary_species, chrom, start, end, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True, temp_file_handler = None ): - if species is not None: alignment = RegionAlignment( end - start, species, temp_file_handler=temp_file_handler ) - else: alignment = RegionAlignment( end - start, primary_species, temp_file_handler=temp_file_handler ) + +# returns a filled region alignment for specified regions +def get_region_alignment( index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None ): + if species is not None: + alignment = RegionAlignment( end - start, species, temp_file_handler=temp_file_handler ) + else: + alignment = RegionAlignment( end - start, primary_species, temp_file_handler=temp_file_handler ) return fill_region_alignment( alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps ) -#reduces a block to only positions exisiting in the src provided + +# reduces a block to only positions exisiting in the src provided def reduce_block_by_primary_genome( block, species, chromosome, region_start ): - #returns ( startIndex, {species:texts} - #where texts' contents are reduced to only positions existing in the primary genome + # returns ( startIndex, {species:texts} + # where texts' contents are reduced to only positions existing in the primary genome src = "%s.%s" % ( species, chromosome ) ref = block.get_component_by_src( src ) start_offset = ref.start - region_start species_texts = {} for c in block.components: species_texts[ c.src.split( '.' )[0] ] = list( c.text ) - #remove locations which are gaps in the primary species, starting from the downstream end + # remove locations which are gaps in the primary species, starting from the downstream end for i in range( len( species_texts[ species ] ) - 1, -1, -1 ): if species_texts[ species ][i] == '-': for text in species_texts.values(): @@ -485,14 +525,15 @@ def reduce_block_by_primary_genome( block, species, chromosome, region_start ): species_texts[spec] = ''.join( text ) return ( start_offset, species_texts ) -#fills a region alignment -def fill_region_alignment( alignment, index, primary_species, chrom, start, end, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True ): + +# fills a region alignment +def fill_region_alignment( alignment, index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True ): region = bx.intervals.Interval( start, end ) region.chrom = chrom region.strand = strand primary_src = "%s.%s" % ( primary_species, chrom ) - #Order blocks overlaping this position by score, lowest first + # Order blocks overlaping this position by score, lowest first blocks = [] for block, idx, offset in index.get_as_iterator_with_index_and_offset( primary_src, start, end ): score = float( block.score ) @@ -503,21 +544,21 @@ def fill_region_alignment( alignment, index, primary_species, chrom, start, end, else: blocks.append( ( score, idx, offset ) ) - #gap_chars_tuple = tuple( GAP_CHARS ) + # gap_chars_tuple = tuple( GAP_CHARS ) gap_chars_str = ''.join( GAP_CHARS ) - #Loop through ordered blocks and layer by increasing score + # Loop through ordered blocks and layer by increasing score for block_dict in blocks: - for block in iter_blocks_split_by_species( block_dict[1].get_at_offset( block_dict[2] ) ): #need to handle each occurance of sequence in block seperately + for block in iter_blocks_split_by_species( block_dict[1].get_at_offset( block_dict[2] ) ): # need to handle each occurance of sequence in block seperately if component_overlaps_region( block.get_component_by_src( primary_src ), region ): - block = chop_block_by_region( block, primary_src, region, species, mincols ) #chop block - block = orient_block_by_region( block, primary_src, region ) #orient block + block = chop_block_by_region( block, primary_src, region, species, mincols ) # chop block + block = orient_block_by_region( block, primary_src, region ) # orient block start_offset, species_texts = reduce_block_by_primary_genome( block, primary_species, chrom, start ) for spec, text in species_texts.items(): - #we should trim gaps from both sides, since these are not positions in this species genome (sequence) + # we should trim gaps from both sides, since these are not positions in this species genome (sequence) text = text.rstrip( gap_chars_str ) gap_offset = 0 - while True in [ text.startswith( gap_char ) for gap_char in GAP_CHARS ]: #python2.4 doesn't accept a tuple for .startswith() - #while text.startswith( gap_chars_tuple ): + # while text.startswith( gap_chars_tuple ): + while True in [ text.startswith( gap_char ) for gap_char in GAP_CHARS ]: # python2.4 doesn't accept a tuple for .startswith() gap_offset += 1 text = text[1:] if not text: @@ -531,48 +572,51 @@ def fill_region_alignment( alignment, index, primary_species, chrom, start, end, alignment.set_position( start_offset + gap_offset + i, spec, char ) return alignment -#returns a filled spliced region alignment for specified region with start and end lists -def get_spliced_region_alignment( index, primary_species, chrom, starts, ends, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True, temp_file_handler = None ): - #create spliced alignment object - if species is not None: alignment = SplicedAlignment( starts, ends, species, temp_file_handler=temp_file_handler ) - else: alignment = SplicedAlignment( starts, ends, [primary_species], temp_file_handler=temp_file_handler ) + +# returns a filled spliced region alignment for specified region with start and end lists +def get_spliced_region_alignment( index, primary_species, chrom, starts, ends, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None ): + # create spliced alignment object + if species is not None: + alignment = SplicedAlignment( starts, ends, species, temp_file_handler=temp_file_handler ) + else: + alignment = SplicedAlignment( starts, ends, [primary_species], temp_file_handler=temp_file_handler ) for exon in alignment.exons: fill_region_alignment( exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps ) return alignment -#loop through string array, only return non-commented lines -def line_enumerator( lines, comment_start = '#' ): + +# loop through string array, only return non-commented lines +def line_enumerator( lines, comment_start='#' ): i = 0 for line in lines: if not line.startswith( comment_start ): i += 1 yield ( i, line ) -#read a GeneBed file, return list of starts, ends, raw fields + +# read a GeneBed file, return list of starts, ends, raw fields def get_starts_ends_fields_from_gene_bed( line ): - #Starts and ends for exons + # Starts and ends for exons starts = [] ends = [] fields = line.split() - #Requires atleast 12 BED columns + # Requires atleast 12 BED columns if len(fields) < 12: raise Exception( "Not a proper 12 column BED line (%s)." % line ) - chrom = fields[0] - tx_start = int( fields[1] ) - tx_end = int( fields[2] ) - name = fields[3] - strand = fields[5] - if strand != '-': strand='+' #Default strand is + + tx_start = int( fields[1] ) + strand = fields[5] + if strand != '-': + strand = '+' # Default strand is + cds_start = int( fields[6] ) - cds_end = int( fields[7] ) + cds_end = int( fields[7] ) - #Calculate and store starts and ends of coding exons + # Calculate and store starts and ends of coding exons region_start, region_end = cds_start, cds_end exon_starts = map( int, fields[11].rstrip( ',\n' ).split( ',' ) ) exon_starts = map( ( lambda x: x + tx_start ), exon_starts ) exon_ends = map( int, fields[10].rstrip( ',' ).split( ',' ) ) - exon_ends = map( ( lambda x, y: x + y ), exon_starts, exon_ends ); + exon_ends = map( ( lambda x, y: x + y ), exon_starts, exon_ends ) for start, end in zip( exon_starts, exon_ends ): start = max( start, region_start ) end = min( end, region_end ) @@ -581,27 +625,33 @@ def get_starts_ends_fields_from_gene_bed( line ): ends.append( end ) return ( starts, ends, fields ) + def iter_components_by_src( block, src ): for c in block.components: if c.src == src: yield c + def get_components_by_src( block, src ): return [ value for value in iter_components_by_src( block, src ) ] + def iter_components_by_src_start( block, src ): for c in block.components: if c.src.startswith( src ): yield c + def get_components_by_src_start( block, src ): return [ value for value in iter_components_by_src_start( block, src ) ] + def sort_block_components_by_block( block1, block2 ): - #orders the components in block1 by the index of the component in block2 - #block1 must be a subset of block2 - #occurs in-place - return block1.components.sort( cmp = lambda x, y: block2.components.index( x ) - block2.components.index( y ) ) + # orders the components in block1 by the index of the component in block2 + # block1 must be a subset of block2 + # occurs in-place + return block1.components.sort( cmp=lambda x, y: block2.components.index( x ) - block2.components.index( y ) ) + def get_species_in_maf( maf_filename ): species = [] @@ -611,20 +661,25 @@ def get_species_in_maf( maf_filename ): species.append( spec ) return species + def parse_species_option( species ): if species: species = species.split( ',' ) if 'None' not in species: return species - return None #provided species was '', None, or had 'None' in it + return None # provided species was '', None, or had 'None' in it + def remove_temp_index_file( index_filename ): - try: os.unlink( index_filename ) - except: pass + try: + os.unlink( index_filename ) + except: + pass -#Below are methods to deal with FASTA files +# Below are methods to deal with FASTA files -def get_fasta_header( component, attributes = {}, suffix = None ): + +def get_fasta_header( component, attributes={}, suffix=None ): header = ">%s(%s):%i-%i|" % ( component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end() ) for key, value in attributes.iteritems(): header = "%s%s=%s|" % ( header, key, value ) @@ -634,8 +689,10 @@ def get_fasta_header( component, attributes = {}, suffix = None ): header = "%s%s" % ( header, src_split( component.src )[ 0 ] ) return header + def get_attributes_from_fasta_header( header ): - if not header: return {} + if not header: + return {} attributes = {} header = header.lstrip( '>' ) header = header.strip() @@ -655,7 +712,7 @@ def get_attributes_from_fasta_header( header ): attributes['start'] = int( region[0] ) attributes['end'] = int( region[1] ) except: - #fields 0 is not a region coordinate + # fields 0 is not a region coordinate pass if len( fields ) > 2: for i in xrange( 1, len( fields ) - 1 ): @@ -666,17 +723,19 @@ def get_attributes_from_fasta_header( header ): attributes['__suffix__'] = fields[-1] return attributes + def iter_fasta_alignment( filename ): class fastaComponent: - def __init__( self, species, text = "" ): + def __init__( self, species, text="" ): self.species = species self.text = text + def extend( self, text ): self.text = self.text + text.replace( '\n', '' ).replace( '\r', '' ).strip() - #yields a list of fastaComponents for a FASTA file + # yields a list of fastaComponents for a FASTA file f = open( filename, 'rb' ) components = [] - #cur_component = None + # cur_component = None while True: line = f.readline() if not line: @@ -693,4 +752,3 @@ def iter_fasta_alignment( filename ): components.append( fastaComponent( attributes['species'] ) ) elif components: components[-1].extend( line ) - diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py index 8cc29a6cfa7..1fc5b75b97a 100644 --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -50,7 +50,7 @@ class RawObjectWrapper( ToolParameterValueWrapper ): try: return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) except: - #Most likely None, which lacks __module__. + # Most likely None, which lacks __module__. return str( self.obj ) def __getattr__( self, key ): @@ -159,7 +159,7 @@ class DatasetFilenameWrapper( ToolParameterValueWrapper ): # again setattr( self, name, rval ) else: - #escape string value of non-defined metadata value + # escape string value of non-defined metadata value rval = wrap_with_safe_string( rval ) return rval @@ -187,7 +187,7 @@ class DatasetFilenameWrapper( ToolParameterValueWrapper ): ext = 'data' self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - # Tool wrappers should not normally be accessing .dataset directly, + # Tool wrappers should not normally be accessing .dataset directly, # so we will wrap it and keep the original around for file paths # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? self.unsanitized = dataset @@ -276,6 +276,7 @@ class DatasetListWrapper( list, ToolParameterValueWrapper, HasDatasets ): return self._dataset_wrapper( dataset, dataset_paths, **kwargs ) list.__init__( self, map( to_wrapper, datasets ) ) + def __str__( self ): return ','.join( map( str, self ) ) diff --git a/lib/galaxy/workflow/extract.py b/lib/galaxy/workflow/extract.py index d539618611a..39c2e57fc2e 100644 --- a/lib/galaxy/workflow/extract.py +++ b/lib/galaxy/workflow/extract.py @@ -279,7 +279,7 @@ class WorkflowSummary( object ): return hdca def __original_hda( self, hda ): - #if this hda was copied from another, we need to find the job that created the origial hda + # if this hda was copied from another, we need to find the job that created the origial hda job_hda = hda while job_hda.copied_from_history_dataset_association: job_hda = job_hda.copied_from_history_dataset_association diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 748579a4d38..d288d64624d 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -5,8 +5,6 @@ Modules used in building workflows import logging import re -from galaxy import eggs - from xml.etree.ElementTree import Element import galaxy.tools @@ -17,8 +15,8 @@ from galaxy.dataset_collections import matching from galaxy.web.framework import formbuilder from galaxy.jobs.actions.post import ActionBox from galaxy.model import PostJobAction -from galaxy.tools.parameters import params_to_incoming, check_param, DataToolParameter, DummyDataset, RuntimeValue, visit_input_values -from galaxy.tools.parameters import DataCollectionToolParameter +from galaxy.tools.parameters import check_param, visit_input_values +from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, DummyDataset, RuntimeValue from galaxy.tools.parameters.wrapped import make_dict_copy from galaxy.tools.execute import execute from galaxy.util.bunch import Bunch @@ -41,7 +39,7 @@ class WorkflowModule( object ): def __init__( self, trans ): self.trans = trans - ## ---- Creating modules from various representations --------------------- + # ---- Creating modules from various representations --------------------- @classmethod def new( Class, trans, tool_id=None ): @@ -62,12 +60,12 @@ class WorkflowModule( object ): def from_workflow_step( Class, trans, step ): return Class( trans ) - ## ---- Saving in various forms ------------------------------------------ + # ---- Saving in various forms ------------------------------------------ def save_to_step( self, step ): step.type = self.type - ## ---- General attributes ----------------------------------------------- + # ---- General attributes ----------------------------------------------- def get_type( self ): return self.type @@ -81,7 +79,7 @@ class WorkflowModule( object ): def get_tooltip( self, static_path='' ): return None - ## ---- Configuration time ----------------------------------------------- + # ---- Configuration time ----------------------------------------------- def get_state( self ): """ Return a serializable representation of the persistable state of @@ -132,7 +130,7 @@ class WorkflowModule( object ): # Replaced connected inputs with DummyDataset values. pass - ## ---- Run time --------------------------------------------------------- + # ---- Run time --------------------------------------------------------- def get_runtime_inputs( self ): """ Used internally by modules and when displaying inputs in workflow @@ -653,7 +651,7 @@ class ToolModule( WorkflowModule ): input_type="dataset_collection", collection_type=input.collection_type, extensions=input.extensions, - ) ) + ) ) visit_input_values( self.tool.inputs, self.state.inputs, callback ) return data_inputs @@ -669,13 +667,13 @@ class ToolModule( WorkflowModule ): formats = [ 'input' ] # TODO: fix elif tool_output.format_source is not None: formats = [ 'input' ] # default to special name "input" which remove restrictions on connections - if data_inputs == None: + if data_inputs is None: data_inputs = self.get_data_inputs() # find the input parameter referenced by format_source for di in data_inputs: # input names come prefixed with conditional and repeat names separated by '|' # remove prefixes when comparing with format_source - if di['name'] != None and di['name'].split('|')[-1] == tool_output.format_source: + if di['name'] is not None and di['name'].split('|')[-1] == tool_output.format_source: formats = di['extensions'] else: formats = [ tool_output.format ] @@ -1048,7 +1046,7 @@ class WorkflowModuleInjector(object): input_connections_by_name = {} for conn in step.input_connections: input_name = conn.input_name - if not input_name in input_connections_by_name: + if input_name not in input_connections_by_name: input_connections_by_name[input_name] = [] input_connections_by_name[input_name].append(conn) step.input_connections_by_name = input_connections_by_name diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py index 5b20d9e8f4b..1dcfae78cb9 100644 --- a/lib/galaxy/workflow/run_request.py +++ b/lib/galaxy/workflow/run_request.py @@ -233,10 +233,10 @@ def build_workflow_run_config( trans, workflow, payload ): trans.security.decode_id(input_id)) assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), content.dataset ) elif input_source == 'uuid': - dataset = trans.sa_session.query(app.model.Dataset).filter(app.model.Dataset.uuid==input_id).first() + dataset = trans.sa_session.query(app.model.Dataset).filter(app.model.Dataset.uuid == input_id).first() if dataset is None: - #this will need to be changed later. If federation code is avalible, then a missing UUID - #could be found amoung fereration partners + # this will need to be changed later. If federation code is avalible, then a missing UUID + # could be found amoung fereration partners message = "Input cannot find UUID: %s." % input_id raise exceptions.RequestParameterInvalidException( message ) assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), dataset ) @@ -330,7 +330,7 @@ def workflow_request_to_run_config( work_request_context, workflow_invocation ): if parameter.name == "copy_inputs_to_history": copy_inputs_to_history = (parameter.value == "true") - #for parameter in workflow_invocation.step_parameters: + # for parameter in workflow_invocation.step_parameters: # step_id = parameter.workflow_step_id # if step_id not in param_map: # param_map[ step_id ] = {} diff --git a/lib/galaxy/workflow/steps.py b/lib/galaxy/workflow/steps.py index 17d13e07a49..ba1ba48a515 100644 --- a/lib/galaxy/workflow/steps.py +++ b/lib/galaxy/workflow/steps.py @@ -30,7 +30,7 @@ def order_workflow_steps( steps ): """ position_data_available = True for step in steps: - if not step.position or not 'left' in step.position or not 'top' in step.position: + if not step.position or 'left' not in step.position or 'top' not in step.position: position_data_available = False if position_data_available: steps.sort(cmp=lambda s1, s2: cmp( math.sqrt(s1.position['left'] ** 2 + s1.position['top'] ** 2), math.sqrt(s2.position['left'] ** 2 + s2.position['top'] ** 2))) diff --git a/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py b/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py index 802dead89ab..fe12c9cce84 100644 --- a/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py +++ b/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py @@ -1,6 +1,10 @@ #!/usr/bin/env python -import os, sys, logging, string, textwrap +import logging +import os +import string +import sys +import textwrap new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) @@ -10,28 +14,29 @@ log = logging.getLogger() log.setLevel( 10 ) log.addHandler( logging.StreamHandler( sys.stdout ) ) -from galaxy import eggs import pkg_resources pkg_resources.require( "SQLAlchemy >= 0.4" ) -import time, ConfigParser, shutil +import time +import ConfigParser from datetime import datetime, timedelta from time import strftime from optparse import OptionParser -from galaxy.tools import parameters from tool_shed.util.common_util import url_join import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping import sqlalchemy as sa -from galaxy.model.orm import and_, not_, distinct +from sqlalchemy import and_, distinct, not_ from galaxy.util import send_mail as galaxy_send_mail assert sys.version_info[:2] >= ( 2, 4 ) + def build_citable_url( host, repository ): return url_join( host, 'view', repository.user.username, repository.name ) + def main(): ''' Script to deprecate any repositories that are older than n days, and have been empty since creation. @@ -42,7 +47,7 @@ def main(): parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", help="verbose mode, print the name of each repository", default=False ) ( options, args ) = parser.parse_args() ini_file = args[0] - config_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} ) + config_parser = ConfigParser.ConfigParser( {'here': os.getcwd()} ) config_parser.read( ini_file ) config_dict = {} for key, value in config_parser.items( "app:main" ): @@ -60,6 +65,7 @@ def main(): deprecate_repositories( app, cutoff_time, days=options.days, info_only=options.info_only, verbose=options.verbose ) + def send_mail_to_owner( app, name, owner, email, repositories_deprecated, days=14 ): ''' Sends an email to the owner of the provided repository. @@ -77,8 +83,8 @@ def send_mail_to_owner( app, name, owner, email, repositories_deprecated, days=1 return subject = "Regarding your tool shed repositories at %s" % url message_body_template = 'The tool shed automated repository checker has discovered that one or more of your repositories hosted ' + \ - 'at this tool shed url ${url} have remained empty for over ${days} days, so they have been marked as deprecated. If you have plans ' + \ - 'for these repositories, you can mark them as un-deprecated at any time.' + 'at this tool shed url ${url} have remained empty for over ${days} days, so they have been marked as deprecated. If you have plans ' + \ + 'for these repositories, you can mark them as un-deprecated at any time.' message_template = string.Template( message_body_template ) body = '\n'.join( textwrap.wrap( message_template.safe_substitute( days=days, url=url ), width=95 ) ) body += '\n\n' @@ -92,12 +98,11 @@ def send_mail_to_owner( app, name, owner, email, repositories_deprecated, days=1 print "# An error occurred attempting to send email: %s" % str( e ) return False + def deprecate_repositories( app, cutoff_time, days=14, info_only=False, verbose=False ): # This method will get a list of repositories that were created on or before cutoff_time, but have never # had any metadata records associated with them. Then it will iterate through that list and deprecate the # repositories, sending an email to each repository owner. - dataset_count = 0 - disk_space = 0 start = time.time() repository_ids_to_not_check = [] # Get a unique list of repository ids from the repository_metadata table. Any repository ID found in this table is not @@ -110,11 +115,11 @@ def deprecate_repositories( app, cutoff_time, days=14, info_only=False, verbose= # Get the repositories that are A) not present in the above list, and b) older than the specified time. # This will yield a list of repositories that have been created more than n days ago, but never populated. repository_query = sa.select( [ app.model.Repository.table.c.id ], - whereclause = and_( app.model.Repository.table.c.create_time < cutoff_time, - app.model.Repository.table.c.deprecated == False, - app.model.Repository.table.c.deleted == False, - not_( app.model.Repository.table.c.id.in_( repository_ids_to_not_check ) ) ), - from_obj = [ app.model.Repository.table ] ) + whereclause=and_( app.model.Repository.table.c.create_time < cutoff_time, + app.model.Repository.table.c.deprecated == False, + app.model.Repository.table.c.deleted == False, + not_( app.model.Repository.table.c.id.in_( repository_ids_to_not_check ) ) ), + from_obj=[ app.model.Repository.table ] ) query_result = repository_query.execute() repositories = [] repositories_by_owner = {} @@ -122,8 +127,7 @@ def deprecate_repositories( app, cutoff_time, days=14, info_only=False, verbose= # Iterate through the list of repository ids for empty repositories and deprecate them unless info_only is set. for repository_id in repository_ids: repository = app.sa_session.query( app.model.Repository ) \ - .filter( app.model.Repository.table.c.id == repository_id ) \ - .one() + .filter( app.model.Repository.table.c.id == repository_id ).one() owner = repository.user if info_only: print '# Repository %s owned by %s would have been deprecated, but info_only was set.' % ( repository.name, repository.user.username ) @@ -147,6 +151,7 @@ def deprecate_repositories( app, cutoff_time, days=14, info_only=False, verbose= print "# Elapsed time: ", stop - start print "####################################################################################" + class DeprecateRepositoriesApplication( object ): """Encapsulates the state of a Universe application""" def __init__( self, config ): @@ -155,6 +160,7 @@ class DeprecateRepositoriesApplication( object ): # Setup the database engine and ORM self.model = galaxy.webapps.tool_shed.model.mapping.init( config.file_path, config.database_connection, engine_options={}, create_tables=False ) self.config = config + @property def sa_session( self ): """ @@ -163,7 +169,9 @@ class DeprecateRepositoriesApplication( object ): to allow migration toward a more SQLAlchemy 0.4 style of use. """ return self.model.context.current + def shutdown( self ): pass -if __name__ == "__main__": main() +if __name__ == "__main__": + main() diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py index 1bfa43d6c4a..fb1a7d4090f 100644 --- a/lib/tool_shed/util/workflow_util.py +++ b/lib/tool_shed/util/workflow_util.py @@ -78,7 +78,7 @@ class RepoToolModule( ToolModule ): data_inputs = [] def callback( input, value, prefixed_name, prefixed_label ): - if isinstance( input, galaxy.tools.parameters.DataToolParameter ): + if isinstance( input, galaxy.tools.parameters.basic.DataToolParameter ): data_inputs.append( dict( name=prefixed_name, label=prefixed_label, extensions=input.extensions ) ) @@ -264,7 +264,7 @@ def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id, # will be ( tool_id, tool_name, tool_version ). missing_tool_tups = [] # First pass to build step objects and populate basic values - for key, step_dict in workflow_dict[ 'steps' ].iteritems(): + for step_dict in workflow_dict[ 'steps' ].itervalues(): # Create the model class for the step step = trans.model.WorkflowStep() step.name = step_dict[ 'name' ] @@ -291,7 +291,7 @@ def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id, step.annotations.append( new_step_annotation ) # Unpack and add post-job actions. post_job_actions = step_dict.get( 'post_job_actions', {} ) - for name, pja_dict in post_job_actions.items(): + for pja_dict in post_job_actions.values(): trans.model.PostJobAction( pja_dict[ 'action_type' ], step, pja_dict[ 'output_name' ], diff --git a/templates/webapps/galaxy/workflow/display.mako b/templates/webapps/galaxy/workflow/display.mako index cadc25b2d4d..367b8118f9c 100644 --- a/templates/webapps/galaxy/workflow/display.mako +++ b/templates/webapps/galaxy/workflow/display.mako @@ -2,8 +2,7 @@ <%namespace file="/display_common.mako" import="render_message" /> <%! - from galaxy.tools.parameters import DataToolParameter, RuntimeValue - from galaxy.tools.parameters import DataCollectionToolParameter + from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue from galaxy.web import form_builder %> diff --git a/templates/webapps/galaxy/workflow/editor_tool_form.mako b/templates/webapps/galaxy/workflow/editor_tool_form.mako index 2466921aab9..f66e07ef19a 100644 --- a/templates/webapps/galaxy/workflow/editor_tool_form.mako +++ b/templates/webapps/galaxy/workflow/editor_tool_form.mako @@ -17,8 +17,7 @@ ${ h.dumps(self.form_config) } %else: <% - from galaxy.tools.parameters import DataToolParameter, RuntimeValue - from galaxy.tools.parameters import DataCollectionToolParameter + from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue from galaxy.util.expressions import ExpressionContext %> @@ -143,4 +142,4 @@ -%endif \ No newline at end of file +%endif diff --git a/templates/webapps/galaxy/workflow/run.mako b/templates/webapps/galaxy/workflow/run.mako index 4e60faad487..8a34ac48ab4 100644 --- a/templates/webapps/galaxy/workflow/run.mako +++ b/templates/webapps/galaxy/workflow/run.mako @@ -301,8 +301,7 @@ <% -from galaxy.tools.parameters import DataToolParameter, RuntimeValue -from galaxy.tools.parameters import DataCollectionToolParameter +from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue from galaxy.jobs.actions.post import ActionBox import re import colorsys diff --git a/test/functional/tools/code_file.py b/test/functional/tools/code_file.py index 5425523e035..a5901294cf5 100644 --- a/test/functional/tools/code_file.py +++ b/test/functional/tools/code_file.py @@ -1,9 +1,3 @@ - - -from galaxy.tools.parameters import DataToolParameter - - - def validate_input( trans, error_map, param_values, page_param_map ): """ Validates the user input, before execution. diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index c170f124800..85271e69212 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -1,21 +1,25 @@ #!/usr/bin/env python -#Processes uploads from the user. +# Processes uploads from the user. # WARNING: Changes in this tool (particularly as related to parsing) may need # to be reflected in galaxy.web.controllers.tool_runner and galaxy.tools -import urllib, sys, os, gzip, tempfile, shutil, re, gzip, zipfile, codecs, binascii -from galaxy import eggs -# need to import model before sniff to resolve a circular import dependency -import galaxy.model -from galaxy.datatypes.checkers import * -from galaxy.datatypes import sniff -from galaxy.datatypes.binary import * -from galaxy.datatypes.images import Pdf -from galaxy.datatypes.registry import Registry +import codecs +import gzip +import os +import shutil +import sys +import tempfile +import urllib +import zipfile + from galaxy import util -from galaxy.datatypes.util.image_util import * -from galaxy.util.json import * +from galaxy.datatypes import sniff +from galaxy.datatypes.binary import Binary +from galaxy.datatypes.checkers import check_binary, check_bz2, check_gzip, check_html, check_image, check_zip +from galaxy.datatypes.registry import Registry +from galaxy.datatypes.util.image_util import get_image_ext +from galaxy.util.json import dumps, loads try: import Image as PIL @@ -32,14 +36,17 @@ except: assert sys.version_info[:2] >= ( 2, 4 ) + def stop_err( msg, ret=1 ): sys.stderr.write( msg ) sys.exit( ret ) + + def file_err( msg, dataset, json_file ): - json_file.write( dumps( dict( type = 'dataset', - ext = 'data', - dataset_id = dataset.dataset_id, - stderr = msg ) ) + "\n" ) + json_file.write( dumps( dict( type='dataset', + ext='data', + dataset_id=dataset.dataset_id, + stderr=msg ) ) + "\n" ) # never remove a server-side upload if dataset.type in ( 'server_dir', 'path_paste' ): return @@ -47,23 +54,29 @@ def file_err( msg, dataset, json_file ): os.remove( dataset.path ) except: pass + + def safe_dict(d): """ Recursively clone json structure with UTF-8 dictionary keys http://mellowmachines.com/blog/2009/06/exploding-dictionary-with-unicode-keys-as-python-arguments/ """ if isinstance(d, dict): - return dict([(k.encode('utf-8'), safe_dict(v)) for k,v in d.iteritems()]) + return dict([(k.encode('utf-8'), safe_dict(v)) for k, v in d.iteritems()]) elif isinstance(d, list): return [safe_dict(x) for x in d] else: return d + + def parse_outputs( args ): rval = {} for arg in args: id, files_path, path = arg.split( ':', 2 ) rval[int( id )] = ( path, files_path ) return rval + + def add_file( dataset, registry, json_file, output_path ): data_type = None line_count = None @@ -80,7 +93,7 @@ def add_file( dataset, registry, json_file, output_path ): if dataset.type == 'url': try: - page = urllib.urlopen( dataset.path ) #page will be .close()ed by sniff methods + page = urllib.urlopen( dataset.path ) # page will be .close()ed by sniff methods temp_name, dataset.is_multi_byte = sniff.stream_to_file( page, prefix='url_paste', source_encoding=util.get_charset_from_http_headers( page.headers ) ) except Exception, e: file_err( 'Unable to fetch %s\n%s' % ( dataset.path, str( e ) ), dataset, json_file ) @@ -133,7 +146,7 @@ def add_file( dataset, registry, json_file, output_path ): elif is_gzipped and is_valid: if link_data_only == 'copy_files': # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format - CHUNK_SIZE = 2**20 # 1Mb + CHUNK_SIZE = 2 ** 20 # 1Mb fd, uncompressed = tempfile.mkstemp( prefix='data_id_%s_upload_gunzip_' % dataset.dataset_id, dir=os.path.dirname( output_path ), text=False ) gzipped_file = gzip.GzipFile( dataset.path, 'rb' ) while 1: @@ -166,7 +179,7 @@ def add_file( dataset, registry, json_file, output_path ): elif is_bzipped and is_valid: if link_data_only == 'copy_files': # We need to uncompress the temp_name file - CHUNK_SIZE = 2**20 # 1Mb + CHUNK_SIZE = 2 ** 20 # 1Mb fd, uncompressed = tempfile.mkstemp( prefix='data_id_%s_upload_bunzip2_' % dataset.dataset_id, dir=os.path.dirname( output_path ), text=False ) bzipped_file = bz2.BZ2File( dataset.path, 'rb' ) while 1: @@ -195,7 +208,7 @@ def add_file( dataset, registry, json_file, output_path ): is_zipped = check_zip( dataset.path ) if is_zipped: if link_data_only == 'copy_files': - CHUNK_SIZE = 2**20 # 1Mb + CHUNK_SIZE = 2 ** 20 # 1Mb uncompressed = None uncompressed_name = None unzipped = False @@ -255,7 +268,7 @@ def add_file( dataset, registry, json_file, output_path ): if check_binary( dataset.path ) or Binary.is_ext_unsniffable(dataset.file_type): # We have a binary dataset, but it is not Bam, Sff or Pdf data_type = 'binary' - #binary_ok = False + # binary_ok = False parts = dataset.name.split( "." ) if len( parts ) > 1: ext = parts[-1].strip().lower() @@ -318,12 +331,12 @@ def add_file( dataset, registry, json_file, output_path ): shutil.move( dataset.path, output_path ) # Write the job info stdout = stdout or 'uploaded %s file' % data_type - info = dict( type = 'dataset', - dataset_id = dataset.dataset_id, - ext = ext, - stdout = stdout, - name = dataset.name, - line_count = line_count ) + info = dict( type='dataset', + dataset_id=dataset.dataset_id, + ext=ext, + stdout=stdout, + name=dataset.name, + line_count=line_count ) if dataset.get('uuid', None) is not None: info['uuid'] = dataset.get('uuid') json_file.write( dumps( info ) + "\n" ) @@ -332,6 +345,7 @@ def add_file( dataset, registry, json_file, output_path ): # Groom the dataset content if necessary datatype.groom_dataset_content( output_path ) + def add_composite_file( dataset, registry, json_file, output_path, files_path ): if dataset.composite_files: os.mkdir( files_path ) @@ -342,15 +356,15 @@ def add_composite_file( dataset, registry, json_file, output_path, files_path ): break elif dataset.composite_file_paths[value.name] is not None: dp = dataset.composite_file_paths[value.name][ 'path' ] - isurl = dp.find('://') <> -1 # todo fixme + isurl = dp.find('://') != -1 # todo fixme if isurl: - try: - temp_name, dataset.is_multi_byte = sniff.stream_to_file( urllib.urlopen( dp ), prefix='url_paste' ) - except Exception, e: - file_err( 'Unable to fetch %s\n%s' % ( dp, str( e ) ), dataset, json_file ) - return - dataset.path = temp_name - dp = temp_name + try: + temp_name, dataset.is_multi_byte = sniff.stream_to_file( urllib.urlopen( dp ), prefix='url_paste' ) + except Exception, e: + file_err( 'Unable to fetch %s\n%s' % ( dp, str( e ) ), dataset, json_file ) + return + dataset.path = temp_name + dp = temp_name if not value.is_binary: tmpdir = output_adjacent_tmpdir( output_path ) tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id @@ -362,9 +376,9 @@ def add_composite_file( dataset, registry, json_file, output_path, files_path ): # Move the dataset to its "real" path shutil.move( dataset.primary_file, output_path ) # Write the job info - info = dict( type = 'dataset', - dataset_id = dataset.dataset_id, - stdout = 'uploaded %s file' % dataset.file_type ) + info = dict( type='dataset', + dataset_id=dataset.dataset_id, + stdout='uploaded %s file' % dataset.file_type ) json_file.write( dumps( info ) + "\n" ) diff --git a/tools/evolution/codingSnps_filter.py b/tools/evolution/codingSnps_filter.py index c428d3ab104..ef2f8a6cd2d 100755 --- a/tools/evolution/codingSnps_filter.py +++ b/tools/evolution/codingSnps_filter.py @@ -1,16 +1,14 @@ #!/usr/bin/env python # runs after the job (and after the default post-filter) -import os -from galaxy import eggs -from galaxy import jobs -from galaxy.tools.parameters import DataToolParameter +from galaxy.tools.parameters.basic import DataToolParameter # Older py compatibility try: set() except: from sets import Set as set + def validate_input( trans, error_map, param_values, page_param_map ): dbkeys = set() data_param_names = set() @@ -18,22 +16,20 @@ def validate_input( trans, error_map, param_values, page_param_map ): for name, param in page_param_map.iteritems(): if isinstance( param, DataToolParameter ): # for each dataset parameter - if param_values.get(name, None) != None: + if param_values.get(name, None) is not None: dbkeys.add( param_values[name].dbkey ) data_params += 1 # check meta data try: param = param_values[name] - startCol = int( param.metadata.startCol ) - endCol = int( param.metadata.endCol ) - chromCol = int( param.metadata.chromCol ) + int( param.metadata.startCol ) + int( param.metadata.endCol ) + int( param.metadata.chromCol ) if param.metadata.strandCol is not None: - strandCol = int ( param.metadata.strandCol ) - else: - strandCol = 0 + int( param.metadata.strandCol ) except: error_msg = "The attributes of this dataset are not properly set. " + \ - "Click the pencil icon in the history item to set the chrom, start, end and strand columns." + "Click the pencil icon in the history item to set the chrom, start, end and strand columns." error_map[name] = error_msg data_param_names.add( name ) if len( dbkeys ) > 1: