opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Open next-stable for the next release.

Browse Source
This commit is contained in:
Nate Coraor
2013-07-30 12:56:17 -04:00
parent c04ccc33d4 caa16ac59e
commit 4b2b0966eb
348 changed files with 20716 additions and 11680 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.coveragerc
+3
View File
@@ -0,0 +1,3 @@
[run]
branch = True
include = lib/galaxy/*
.hgignore
+4 -1
View File
@@ -60,7 +60,7 @@ shed_tool_data_table_conf.xml
job_conf.xml
data_manager_conf.xml
shed_data_manager_conf.xml
visualizations_conf.xml
config/visualizations/*.xml
static/welcome.html.*
static/welcome.html
@@ -82,6 +82,9 @@ tool-data/genome/*
# Test output
run_functional_tests.html
test/tool_shed/tmp/*
.coverage
htmlcov
run_unit_tests.html
# Project files
*.kpf
.hgtags
+1
View File
@@ -2,3 +2,4 @@ a4113cc1cb5eaa68091c9a73375f00555b66dd11 release_2013.01.13
1c717491139269651bb59687563da9410b84c65d release_2013.02.08
75f09617abaadbc8cc732bb8ee519decaeb56ea7 release_2013.04.01
2cc8d10988e03257dc7b97f8bb332c7df745d1dd security_2013.04.08
524f246ca85395082719ae7a6ff72260d7ad5612 release_2013.06.03
README.txt
+1 -1
View File
@@ -7,7 +7,7 @@ website above.
HOW TO START
============
Galaxy requires Python 2.5, 2.6 or 2.7. To check your python version, run:
Galaxy requires Python 2.6 or 2.7. To check your python version, run:
% python -V
Python 2.7.3
buildbot_setup.sh
+1
View File
@@ -65,6 +65,7 @@ LINKS="
"
SAMPLES="
tool_conf.xml.sample
datatypes_conf.xml.sample
universe_wsgi.ini.sample
tool_data_table_conf.xml.sample
config/visualizations/circster.xml.sample
+26
View File
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE visualization SYSTEM "visualization.dtd">
<visualization name="circster">
<data_sources>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test>
<to_param param_attr="id">dataset_id</to_param>
<to_param assign="hda">hda_ldda</to_param>
</data_source>
<data_source>
<model_class>LibraryDatasetDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test>
<to_param param_attr="id">dataset_id</to_param>
<to_param assign="ldda">hda_ldda</to_param>
</data_source>
</data_sources>
<params>
<param type="visualization">id</param>
<param type="hda_or_ldda">dataset_id</param>
<param_modifier type="string" modifies="dataset_id">hda_ldda</param_modifier>
<param type="dbkey">dbkey</param>
</params>
<template>circster.mako</template>
<render_location>_top</render_location>
</visualization>
config/visualizations/phyloviz.xml.sample
+18
View File
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE visualization SYSTEM "visualization.dtd">
<visualization name="phyloviz">
<data_sources>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Newick</test>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Nexus</test>
<to_param param_attr="id">dataset_id</to_param>
</data_source>
</data_sources>
<params>
<param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
<param type="integer" default="0">tree_index</param>
</params>
<template>phyloviz.mako</template>
<render_location>_top</render_location>
</visualization>
config/visualizations/scatterplot.xml.sample
+15
View File
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE visualization SYSTEM "visualization.dtd">
<visualization name="scatterplot">
<data_sources>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
<to_param param_attr="id">dataset_id</to_param>
</data_source>
</data_sources>
<params>
<param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
</params>
<template>scatterplot.mako</template>
</visualization>
config/visualizations/sweepster.xml.sample
+25
View File
@@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE visualization SYSTEM "visualization.dtd">
<visualization name="sweepster">
<data_sources>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test>
<to_param param_attr="id">dataset_id</to_param>
<to_param assign="hda">hda_ldda</to_param>
</data_source>
<data_source>
<model_class>LibraryDatasetDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test>
<to_param param_attr="id">dataset_id</to_param>
<to_param assign="ldda">hda_ldda</to_param>
</data_source>
</data_sources>
<params>
<param type="visualization" var_name_in_template="viz">visualization</param>
<param type="hda_or_ldda" var_name_in_template="dataset">dataset_id</param>
<param_modifier type="string" modifies="dataset_id">hda_ldda</param_modifier>
</params>
<template>sweepster.mako</template>
<render_location>_top</render_location>
</visualization>
config/visualizations/trackster.xml.sample
+29
View File
@@ -0,0 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE visualization SYSTEM "visualization.dtd">
<visualization name="trackster">
<!--not tested yet -->
<data_sources>
<data_source>
<model_class>HistoryDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test>
<to_param param_attr="id">dataset_id</to_param>
<to_param assign="hda">hda_ldda</to_param>
<to_param param_attr="dbkey">dbkey</to_param>
</data_source>
<data_source>
<model_class>LibraryDatasetDatasetAssociation</model_class>
<test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test>
<to_param param_attr="id">dataset_id</to_param>
<to_param assign="ldda">hda_ldda</to_param>
</data_source>
</data_sources>
<params>
<param type="visualization">id</param>
<param type="dataset">dataset_id</param>
<param type="genome_region">genome_region</param>
<param type="dbkey">dbkey</param>
</params>
<template_root>tracks</template_root>
<template>browser.mako</template>
<render_location>_top</render_location>
</visualization>
config/visualizations/visualization.dtd
+132
View File
@@ -0,0 +1,132 @@
<!-- runnable on NIX with xmllint -->
<!-- each visualization must have a template (all other elements are optional) -->
<!ELEMENT visualization (data_sources*,params*,template_root*,template,link_text*,render_location*)>
<!-- visualization name (e.g. 'trackster', 'scatterplot', etc.) is required -->
<!ATTLIST visualization
name CDATA #REQUIRED
>
<!ELEMENT data_sources (data_source*)>
<!-- data sources are elements that describe what objects (HDAs, LDDAs, Job, User, etc.)
are applicable to a visualization. Often these are used to fetch applicable links
to the visualizations that use them.
-->
<!ELEMENT data_source (model_class,(test|to_param)*)>
<!ELEMENT model_class (#PCDATA)>
<!-- model_class is currently the class name of the object you want to make a visualization
applicable to (e.g. HistoryDatasetAssociation). Currently only classes in galaxy.model
can be used.
REQUIRED and currently limited to: 'HistoryDatasetAssociation', 'LibraryDatasetDatasetAssociation'
-->
<!ELEMENT test (#PCDATA)>
<!-- tests help define what conditions the visualization can be applied to the model_class/target.
Currently, all tests are OR'd and there is no logical grouping. Tests are run in order.
(text): the text of this element is what the given target will be compared to (REQUIRED)
type: what type of test to run (e.g. when the target is an HDA the test will often be of type 'isinstance'
and test whether the HDA's datatype isinstace of a class)
DEFAULT: string comparison.
test_attr: what attribute of the target object should be used in the test. For instance, 'datatype'
will attempt to get the HDA.datatype from a target HDA. If the given object doesn't have
that attribute the test will fail (with no error). test_attr can be dot separated attributes,
looking up each in turn. For example, if the target was a history, one could access the
history.user.email by setting test_attr to 'user.email' (why you would want that, I don't know)
DEFAULT: to comparing the object itself (and not any of it's attributes)
result_type: if the result (the text of the element mentioned above) needs to be parsed into
something other than a string, result_type will tell the registry how to do this. E.g.
if result_type is 'datatype' the registry will assume the text is a datatype class name
and parse it into the proper class before the test (often 'isinstance') is run.
DEFAULT: no parsing (result should be a string)
-->
<!ATTLIST test
type CDATA #IMPLIED
test_attr CDATA #IMPLIED
result_type CDATA #IMPLIED
>
<!ELEMENT to_param (#PCDATA)>
<!-- to_param tells the registry how to parse the data_source into a query string param.
For example, HDA data_sources can set param_to text to 'dataset_id' and param_attr to 'id' and the
the target HDA (if it passes the tests) will be passed as "dataset_id=HDA.id"
(text): the query string param key this source will be parsed into (e.g. dataset_id)
REQUIRED
param_attr: the attribute of the data_source object to use as the value in the query string param.
E.g. param_attr='id' for an HDA data_source would use the (encoded) id.
NOTE: a to_param MUST have either a param_attr or assign
assign: you can use this to directly assign a value to a query string's param. E.g. if the
data_source is a LDDA we can set 'hda_or_ldda=ldda' using assign='ldda'.
NOTE: a to_param MUST have either a param_attr or assign
-->
<!ATTLIST to_param
param_attr CDATA #IMPLIED
assign CDATA #IMPLIED
>
<!ELEMENT params ((param|param_modifier)*)>
<!-- params describe what data will be sent to a visualization template and
how to convert them from a query string in a URL into variables usable in a template.
For example,
param_modifiers are a special class of parameters that modify other params
(e.g. hda_ldda can be 'hda' or 'ldda' and modifies/informs dataset_id to fetch an HDA or LDDA)
-->
<!ELEMENT param (#PCDATA)>
<!-- param tells the registry how to parse the query string param back into a resource/data_source.
For example, if a query string has "dataset_id=NNN" and the type is 'dataset', the registry
will attempt to fetch the hda with id of NNN from the database and pass it to the template.
(text): the query string param key this source will be parsed from (e.g. dataset_id)
REQUIRED
type: the type of the resource.
Can be: str (DEFAULT), bool, int, float, json, visualization, dbkey, dataset, or hda_ldda.
default: if a param is not passed on the query string (and is not required) OR the given param
fails to parse, this value is used instead.
DEFAULT: None
required: set this to true if the param is required for the template. Rendering will with an error
if the param hasn't been sent.
DEFAULT: false
csv: set this to true if the param is a comma separated list. The registry will attempt to
parse each value as the given type and send the result as a list to the template.
DEFAULT: false
constrain_to: (currently unused) constain a param to a set of values, error if not valid.
DEFAULT: don't constrain
var_name_in_template: a new name for the resource/variable to use in the template. E.g. an initial
query string param key might be 'dataset_id' in the URL, the registry parses it into an HDA,
and if var_name_in_template is set to 'hda', the template will be able to access the HDA
with the variable name 'hda' (as in hda.title).
DEFAULT: keep the original query string name
-->
<!ATTLIST param
type CDATA #IMPLIED
default CDATA #IMPLIED
required CDATA #IMPLIED
csv CDATA #IMPLIED
constrain_to CDATA #IMPLIED
var_name_in_template CDATA #IMPLIED
>
<!-- param_modifiers are the same as param but have a REQUIRED 'modifies' attribute.
'modifies' must point to the param name (the text part of param element) that it will modify.
E.g. <param_modifier modifies="dataset_id">hda_ldda</param_modifier>
-->
<!ELEMENT param_modifier (#PCDATA)>
<!ATTLIST param_modifier
modifies CDATA #REQUIRED
type CDATA #IMPLIED
default CDATA #IMPLIED
required CDATA #IMPLIED
csv CDATA #IMPLIED
constrain_to CDATA #IMPLIED
var_name_in_template CDATA #IMPLIED
>
<!-- template_root: the directory to search for the template relative to templates/webapps/galaxy
(optional) DEFAULT: visualizations
-->
<!ELEMENT template_root (#PCDATA)>
<!-- template: the template used to render the visualization. REQUIRED -->
<!ELEMENT template (#PCDATA)>
<!-- link_text: the text component of an html anchor displayed when the registry builds the link information -->
<!ELEMENT link_text (#PCDATA)>
<!-- render_location: used as the target attribute of the link to the visualization.
Can be 'galaxy_main', '_top', '_blank'. DEFAULT: 'galaxy_main'
-->
<!-- TODO: rename -> render_target -->
<!ELEMENT render_location (#PCDATA)>
eggs.ini
+1 -1
View File
@@ -19,7 +19,7 @@ MarkupSafe = 0.12
mercurial = 2.2.3
MySQL_python = 1.2.3c1
numpy = 1.6.0
pbs_python = 4.1.0
pbs_python = 4.3.5
psycopg2 = 2.0.13
pycrypto = 2.5
pysam = 0.4.2
job_conf.xml.sample_advanced
+9 -1
View File
@@ -54,7 +54,15 @@
<param id="shell_hostname">foo.example.org</param>
<param id="Job_Execution_Time">24:00:00</param>
</destination>
<destination id="condor" runner="condor"/>
<destination id="condor" runner="condor">
<!-- With no params, jobs are submitted to the 'vanilla' universe with:
notification = NEVER
getenv = true
Additional/override query ClassAd params can be specified with
<param> tags.
-->
<param id="request_cpus">8</param>
</destination>
</destinations>
<tools>
<!-- Tools can be configured to use specific destinations or handlers,
lib/galaxy/app.py
+5 -3
View File
@@ -92,7 +92,7 @@ class UniverseApplication( object ):
# Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
tool_data_path=self.tool_data_tables.tool_data_path,
from_shed_config=True )
from_shed_config=False )
# Initialize the job management configuration
self.job_config = jobs.JobConfiguration(self)
# Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
@@ -123,8 +123,10 @@ class UniverseApplication( object ):
# Load genome indexer tool.
load_genome_index_tools( self.toolbox )
# visualizations registry: associates resources with visualizations, controls how to render
self.visualizations_registry = ( VisualizationsRegistry( self.config.root, self.config.visualizations_conf_path )
if self.config.visualizations_conf_path else None )
self.visualizations_registry = None
if self.config.visualizations_config_directory:
self.visualizations_registry = VisualizationsRegistry( self.config.root,
self.config.visualizations_config_directory )
# Load security policy.
self.security_agent = self.model.security_agent
self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions )
lib/galaxy/config.py
+22 -9
View File
@@ -64,21 +64,33 @@ class Configuration( object ):
tcf = kwargs[ 'tool_config_files' ]
else:
tcf = 'tool_conf.xml'
self.tool_filters = listify( kwargs.get( "tool_filters", [] ) )
self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ) )
self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ) )
self.tool_configs = [ resolve_path( p, self.root ) for p in listify( tcf ) ]
self.shed_tool_data_path = kwargs.get( "shed_tool_data_path", None )
if self.shed_tool_data_path:
self.shed_tool_data_path = resolve_path( self.shed_tool_data_path, self.root )
else:
self.shed_tool_data_path = self.tool_data_path
self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root )
self.shed_tool_data_table_config = resolve_path( kwargs.get( 'shed_tool_data_table_config', 'shed_tool_data_table_conf.xml' ), self.root )
self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) )
self.hours_between_check = kwargs.get( 'hours_between_check', 12 )
try:
self.hours_between_check = kwargs.get( 'hours_between_check', 12 )
if isinstance( self.hours_between_check, float ):
hbc_test = int( self.hours_between_check )
self.hours_between_check = hbc_test
if self.hours_between_check < 1 or self.hours_between_check > 24:
self.hours_between_check = 12
except:
try:
# Float values are supported for functional tests.
hbc_test = float( self.hours_between_check )
self.hours_between_check = hbc_test
if self.hours_between_check < 0.001 or self.hours_between_check > 24.0:
self.hours_between_check = 12.0
else:
if self.hours_between_check < 1 or self.hours_between_check > 24:
self.hours_between_check = 12
except:
self.hours_between_check = 12
except:
self.hours_between_check = 12
self.update_integrated_tool_panel = kwargs.get( "update_integrated_tool_panel", True )
self.enable_data_manager_user_view = string_as_bool( kwargs.get( "enable_data_manager_user_view", "False" ) )
self.data_manager_config_file = resolve_path( kwargs.get('data_manager_config_file', 'data_manager_conf.xml' ), self.root )
@@ -154,6 +166,7 @@ class Configuration( object ):
self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla" ).lower().split(",")
self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225" ).lower().split(",")
self.brand = kwargs.get( 'brand', None )
self.welcome_url = kwargs.get( 'welcome_url', '/static/welcome.html' )
# Configuration for the message box directly below the masthead.
self.message_box_visible = kwargs.get( 'message_box_visible', False )
self.message_box_content = kwargs.get( 'message_box_content', None )
@@ -275,8 +288,8 @@ class Configuration( object ):
self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) )
self.fluent_host = kwargs.get( 'fluent_host', 'localhost' )
self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) )
# visualizations registry config path
self.visualizations_conf_path = kwargs.get( 'visualizations_conf_path', None )
# visualization registries config directory
self.visualizations_config_directory = kwargs.get( 'visualizations_config_directory', None )
@property
def sentry_dsn_public( self ):
lib/galaxy/datatypes/binary.py
+95
View File
@@ -22,6 +22,7 @@ from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes import metadata
from galaxy.datatypes.sniff import *
import dataproviders
log = logging.getLogger(__name__)
@@ -74,6 +75,7 @@ class Binary( data.Data ):
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext)
return open( dataset.file_name )
class Ab1( Binary ):
"""Class describing an ab1 binary sequence file"""
file_ext = "ab1"
@@ -93,12 +95,15 @@ class Ab1( Binary ):
Binary.register_unsniffable_binary_ext("ab1")
class GenericAsn1Binary( Binary ):
"""Class for generic ASN.1 binary format"""
file_ext = "asn1-binary"
Binary.register_unsniffable_binary_ext("asn1-binary")
@dataproviders.decorators.has_dataproviders
class Bam( Binary ):
"""Class describing a BAM binary file"""
file_ext = "bam"
@@ -255,9 +260,92 @@ class Bam( Binary ):
return dataset.peek
except:
return "Binary bam alignments file (%s)" % ( data.nice_size( dataset.get_size() ) )
# ------------- Dataproviders
# pipe through samtools view
#ALSO: (as Sam)
# bam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers
#TODO:?? seems like there should be an easier way to do/inherit this - metadata.comment_char?
#TODO: incorporate samtools options to control output: regions first, then flags, etc.
@dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings )
def line_dataprovider( self, dataset, **settings ):
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
settings[ 'comment_char' ] = '@'
return dataproviders.line.FilteredLineDataProvider( samtools_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'regex-line', dataproviders.line.RegexLineDataProvider.settings )
def regex_line_dataprovider( self, dataset, **settings ):
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
settings[ 'comment_char' ] = '@'
return dataproviders.line.RegexLineDataProvider( samtools_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings )
def column_dataprovider( self, dataset, **settings ):
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
settings[ 'comment_char' ] = '@'
return dataproviders.column.ColumnarDataProvider( samtools_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'dict', dataproviders.column.DictDataProvider.settings )
def dict_dataprovider( self, dataset, **settings ):
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
settings[ 'comment_char' ] = '@'
return dataproviders.column.DictDataProvider( samtools_source, **settings )
# these can't be used directly - may need BamColumn, BamDict (Bam metadata -> column/dict)
# OR - see genomic_region_dataprovider
#@dataproviders.decorators.dataprovider_factory( 'dataset-column', dataproviders.column.ColumnarDataProvider.settings )
#def dataset_column_dataprovider( self, dataset, **settings ):
# settings[ 'comment_char' ] = '@'
# return super( Sam, self ).dataset_column_dataprovider( dataset, **settings )
#@dataproviders.decorators.dataprovider_factory( 'dataset-dict', dataproviders.column.DictDataProvider.settings )
#def dataset_dict_dataprovider( self, dataset, **settings ):
# settings[ 'comment_char' ] = '@'
# return super( Sam, self ).dataset_dict_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'header', dataproviders.line.RegexLineDataProvider.settings )
def header_dataprovider( self, dataset, **settings ):
# in this case we can use an option of samtools view to provide just what we need (w/o regex)
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset, '-H' )
return dataproviders.line.RegexLineDataProvider( samtools_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'id-seq-qual', dataproviders.column.DictDataProvider.settings )
def id_seq_qual_dataprovider( self, dataset, **settings ):
settings[ 'indeces' ] = [ 0, 9, 10 ]
settings[ 'column_types' ] = [ 'str', 'str', 'str' ]
settings[ 'column_names' ] = [ 'id', 'seq', 'qual' ]
return self.dict_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region', dataproviders.column.ColumnarDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
# GenomicRegionDataProvider currently requires a dataset as source - may not be necc.
#TODO:?? consider (at least) the possible use of a kwarg: metadata_source (def. to source.dataset),
# or remove altogether...
#samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
#return dataproviders.dataset.GenomicRegionDataProvider( samtools_source, metadata_source=dataset,
# 2, 3, 3, **settings )
# instead, set manually and use in-class column gen
settings[ 'indeces' ] = [ 2, 3, 3 ]
settings[ 'column_types' ] = [ 'str', 'int', 'int' ]
return self.column_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-dict', dataproviders.column.DictDataProvider.settings )
def genomic_region_dict_dataprovider( self, dataset, **settings ):
settings[ 'indeces' ] = [ 2, 3, 3 ]
settings[ 'column_types' ] = [ 'str', 'int', 'int' ]
settings[ 'column_names' ] = [ 'chrom', 'start', 'end' ]
return self.dict_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'samtools' )
def samtools_dataprovider( self, dataset, **settings ):
"""Generic samtools interface - all options available through settings."""
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.dataset.SamtoolsDataProvider( dataset_source, **settings )
Binary.register_sniffable_binary_format("bam", "bam", Bam)
class H5( Binary ):
"""Class describing an HDF5 file"""
file_ext = "h5"
@@ -277,6 +365,7 @@ class H5( Binary ):
Binary.register_unsniffable_binary_ext("h5")
class Scf( Binary ):
"""Class describing an scf binary sequence file"""
file_ext = "scf"
@@ -296,6 +385,7 @@ class Scf( Binary ):
Binary.register_unsniffable_binary_ext("scf")
class Sff( Binary ):
""" Standard Flowgram Format (SFF) """
file_ext = "sff"
@@ -327,6 +417,7 @@ class Sff( Binary ):
Binary.register_sniffable_binary_format("sff", "sff", Sff)
class BigWig(Binary):
"""
Accessing binary BigWig files from UCSC.
@@ -363,6 +454,7 @@ class BigWig(Binary):
Binary.register_sniffable_binary_format("bigwig", "bigwig", BigWig)
class BigBed(BigWig):
"""BigBed support from UCSC."""
@@ -375,6 +467,7 @@ class BigBed(BigWig):
Binary.register_sniffable_binary_format("bigbed", "bigbed", BigBed)
class TwoBit (Binary):
"""Class describing a TwoBit format nucleotide file"""
@@ -399,3 +492,5 @@ class TwoBit (Binary):
return dataset.peek
except:
return "Binary TwoBit format nucleotide file (%s)" % (data.nice_size(dataset.get_size()))
Binary.register_sniffable_binary_format("twobit", "twobit", TwoBit)
lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
+4
View File
@@ -1,5 +1,9 @@
<tool id="CONVERTER_bam_to_bigwig_0" name="Convert BAM to BigWig" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
<requirement type="package">bedtools</requirement>
</requirements>
<command>
bedtools genomecov -bg -split -ibam $input -g $chromInfo
lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
+4
View File
@@ -1,5 +1,9 @@
<tool id="CONVERTER_bed_gff_or_vcf_to_bigwig_0" name="Convert BED, GFF, or VCF to BigWig" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
<requirement type="package">bedtools</requirement>
</requirements>
<command>
## Remove comments and sort by chromosome.
grep -v '^#' $input | sort -k1,1 |
lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
+4
View File
@@ -1,6 +1,10 @@
<tool id="CONVERTER_interval_to_bigwig_0" name="Convert Genomic Intervals To Coverage">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
<requirement type="package">bedtools</requirement>
</requirements>
<command>
## Remove comments and sort by chromosome.
lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py
+8 -10
View File
@@ -11,15 +11,13 @@ usage: %prog <options> in_file out_file
from __future__ import division
import sys, fileinput, optparse
import optparse
from galaxy import eggs
import pkg_resources; pkg_resources.require( "bx-python" )
from galaxy.visualization.tracks.summary import *
from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed
eggs.require( "bx-python" )
from bx.interval_index_file import Indexes
def main():
# Read options, args.
parser = optparse.OptionParser()
parser.add_option( '-c', '--chr-col', type='int', dest='chrom_col', default=1 )
@@ -27,12 +25,12 @@ def main():
parser.add_option( '-e', '--end-col', type='int', dest='end_col', default=3 )
(options, args) = parser.parse_args()
input_fname, output_fname = args
# Make column indices 0-based.
options.chrom_col -= 1
options.start_col -= 1
options.end_col -= 1
# Do conversion.
index = Indexes()
offset = 0
@@ -46,9 +44,9 @@ def main():
chrom_end = int( feature[ options.end_col ] )
index.add( chrom, chrom_start, chrom_end, offset )
offset += len(line)
index.write( open(output_fname, "w") )
if __name__ == "__main__":
if __name__ == "__main__":
main()
lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py
+7 -9
View File
@@ -8,20 +8,18 @@ usage: %prog <options> in_file out_file
from __future__ import division
import sys, fileinput, optparse
import optparse
from galaxy import eggs
import pkg_resources; pkg_resources.require( "bx-python" )
from galaxy.visualization.tracks.summary import *
from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed
eggs.require( "bx-python" )
from bx.interval_index_file import Indexes
def main():
# Read options, args.
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
input_fname, output_fname = args
# Do conversion.
index = Indexes()
offset = 0
@@ -31,9 +29,9 @@ def main():
start = int( start ) - 1
index.add( chrom, start, start + 1, offset )
offset += len( line )
index.write( open(output_fname, "w") )
if __name__ == "__main__":
if __name__ == "__main__":
main()
lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
+5
View File
@@ -1,4 +1,9 @@
<tool id="CONVERTER_sam_to_bigwig_0" name="Convert SAM to BigWig" version="1.0.0" hidden="true">
<requirements>
<requirement type="package">ucsc_tools</requirement>
<requirement type="package">samtools</requirement>
<requirement type="package">bedtools</requirement>
</requirements>
<command>
samtools view -bh $input | bedtools genomecov -bg -split -ibam stdin -g $chromInfo
lib/galaxy/datatypes/data.py
+65 -1
View File
@@ -14,6 +14,8 @@ from galaxy.util.bunch import Bunch
from galaxy.util.odict import odict
from galaxy.util.sanitize_html import sanitize_html
import dataproviders
from galaxy import eggs
eggs.require( "Paste" )
import paste
@@ -56,6 +58,7 @@ class DataMeta( type ):
cls.metadata_spec.update( base.metadata_spec ) #add contents of metadata spec of base class to cls
metadata.Statement.process( cls )
@dataproviders.decorators.has_dataproviders
class Data( object ):
"""
Base class for all datatypes. Implements basic interfaces as well
@@ -545,7 +548,13 @@ class Data( object ):
def has_resolution(self):
return False
def matches_any( self, target_datatypes ):
"""
Check if this datatype is of any of the target_datatypes or is
a subtype thereof.
"""
datatype_classes = tuple( [ datatype.__class__ for datatype in target_datatypes ] )
return isinstance( self, datatype_classes )
def merge( split_files, output_file):
"""
Merge files with copy.copyfileobj() will not hit the
@@ -572,6 +581,39 @@ class Data( object ):
return [ 'trackster', 'circster' ]
return []
# ------------- Dataproviders
def has_dataprovider( self, data_format ):
"""
Returns True if `data_format` is available in `dataproviders`.
"""
return ( data_format in self.dataproviders )
def dataprovider( self, dataset, data_format, **settings ):
"""
Base dataprovider factory for all datatypes that returns the proper provider
for the given `data_format` or raises a `NoProviderAvailable`.
"""
if self.has_dataprovider( data_format ):
return self.dataproviders[ data_format ]( self, dataset, **settings )
raise dataproviders.exceptions.NoProviderAvailable( self, data_format )
@dataproviders.decorators.dataprovider_factory( 'base' )
def base_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.base.DataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'chunk', dataproviders.chunk.ChunkDataProvider.settings )
def chunk_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.chunk.ChunkDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'chunk64', dataproviders.chunk.Base64ChunkDataProvider.settings )
def chunk64_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.chunk.Base64ChunkDataProvider( dataset_source, **settings )
@dataproviders.decorators.has_dataproviders
class Text( Data ):
file_ext = 'txt'
line_class = 'line'
@@ -741,10 +783,31 @@ class Text( Data ):
f.close()
split = classmethod(split)
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings )
def line_dataprovider( self, dataset, **settings ):
"""
Returns an iterator over the dataset's lines (that have been `strip`ed)
optionally excluding blank lines and lines that start with a comment character.
"""
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.line.FilteredLineDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'regex-line', dataproviders.line.RegexLineDataProvider.settings )
def regex_line_dataprovider( self, dataset, **settings ):
"""
Returns an iterator over the dataset's lines
optionally including/excluding lines that match one or more regex filters.
"""
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.line.RegexLineDataProvider( dataset_source, **settings )
class GenericAsn1( Text ):
"""Class for generic ASN.1 text format"""
file_ext = 'asn1'
class LineCount( Text ):
"""
Dataset contains a single line with a single integer that denotes the
@@ -752,6 +815,7 @@ class LineCount( Text ):
"""
pass
class Newick( Text ):
"""New Hampshire/Newick Format"""
file_ext = "nhx"
lib/galaxy/datatypes/dataproviders/__init__.py
+28
View File
@@ -0,0 +1,28 @@
#TODO: ---- This is a work in progress ----
"""
Dataproviders are iterators with context managers that provide data to some
consumer datum by datum.
As well as subclassing and overriding to get the proper data, Dataproviders
can be piped from one to the other.
..example::
.. note:: be careful to NOT pipe providers into subclasses of those providers.
Subclasses provide all the functionality of their superclasses,
so there's generally no need.
.. note:: be careful to when using piped providers that accept the same keywords
in their __init__ functions (such as limit or offset) to pass those
keywords to the proper (often final) provider. These errors that result
can be hard to diagnose.
"""
import decorators
import exceptions
import base
import chunk
import line
import column
import external
import dataset
lib/galaxy/datatypes/dataproviders/base.py
+305
View File
@@ -0,0 +1,305 @@
"""
Base class(es) for all DataProviders.
"""
# there's a blurry line between functionality here and functionality in datatypes module
# attempting to keep parsing to a minimum here and focus on chopping/pagination/reformat(/filtering-maybe?)
# and using as much pre-computed info/metadata from the datatypes module as possible
# also, this shouldn't be a replacement/re-implementation of the tool layer
# (which provides traceability/versioning/reproducibility)
from collections import deque
import exceptions
_TODO = """
hooks into datatypes (define providers inside datatype modules) as factories
capture tell() when provider is done
def stop( self ): self.endpoint = source.tell(); raise StopIteration()
implement __len__ sensibly where it can be (would be good to have where we're giving some progress - '100 of 300')
seems like sniffed files would have this info
unit tests
add datum entry/exit point methods: possibly decode, encode
or create a class that pipes source through - how would decode work then?
icorporate existing visualization/dataproviders
some of the sources (esp. in datasets) don't need to be re-created
YAGNI: InterleavingMultiSourceDataProvider, CombiningMultiSourceDataProvider
datasets API entry point:
kwargs should be parsed from strings 2 layers up (in the DatasetsAPI) - that's the 'proper' place for that.
but how would it know how/what to parse if it doesn't have access to the classes used in the provider?
Building a giant list by sweeping all possible dprov classes doesn't make sense
For now - I'm burying them in the class __init__s - but I don't like that
"""
import logging
log = logging.getLogger( __name__ )
# ----------------------------------------------------------------------------- base classes
class HasSettings( type ):
"""
Metaclass for data providers that allows defining and inheriting
a dictionary named 'settings'.
Useful for allowing class level access to expected variable types
passed to class `__init__` functions so they can be parsed from a query string.
"""
# yeah - this is all too acrobatic
def __new__( cls, name, base_classes, attributes ):
settings = {}
# get settings defined in base classes
for base_class in base_classes:
base_settings = getattr( base_class, 'settings', None )
if base_settings:
settings.update( base_settings )
# get settings defined in this class
new_settings = attributes.pop( 'settings', None )
if new_settings:
settings.update( new_settings )
attributes[ 'settings' ] = settings
return type.__new__( cls, name, base_classes, attributes )
# ----------------------------------------------------------------------------- base classes
class DataProvider( object ):
"""
Base class for all data providers. Data providers:
(a) have a source (which must be another file-like object)
(b) implement both the iterator and context manager interfaces
(c) do not allow write methods
(but otherwise implement the other file object interface methods)
"""
# a definition of expected types for keyword arguments sent to __init__
# useful for controlling how query string dictionaries can be parsed into correct types for __init__
# empty in this base class
__metaclass__ = HasSettings
settings = {}
def __init__( self, source, **kwargs ):
"""
:param source: the source that this iterator will loop over.
(Should implement the iterable interface and ideally have the
context manager interface as well)
"""
self.source = self.validate_source( source )
def validate_source( self, source ):
"""
Is this a valid source for this provider?
:raises InvalidDataProviderSource: if the source is considered invalid.
Meant to be overridden in subclasses.
"""
if not source or not hasattr( source, '__iter__' ):
# that's by no means a thorough check
raise exceptions.InvalidDataProviderSource( source )
return source
#TODO: (this might cause problems later...)
#TODO: some providers (such as chunk's seek and read) rely on this... remove
def __getattr__( self, name ):
if name == 'source':
# if we're inside this fn, source hasn't been set - provide some safety just for this attr
return None
# otherwise, try to get the attr from the source - allows us to get things like provider.encoding, etc.
if hasattr( self.source, name ):
return getattr( self.source, name )
# raise the proper error
return self.__getattribute__( name )
# write methods should not be allowed
def truncate( self, size ):
raise NotImplementedError( 'Write methods are purposely disabled' )
def write( self, string ):
raise NotImplementedError( 'Write methods are purposely disabled' )
def writelines( self, sequence ):
raise NotImplementedError( 'Write methods are purposely disabled' )
#TODO: route read methods through next?
#def readline( self ):
# return self.next()
def readlines( self ):
return [ line for line in self ]
# iterator interface
def __iter__( self ):
# it's generators all the way up, Timmy
with self as source:
for datum in self.source:
yield datum
def next( self ):
return self.source.next()
# context manager interface
def __enter__( self ):
# make the source's context manager interface optional
if hasattr( self.source, '__enter__' ):
self.source.__enter__()
return self
def __exit__( self, *args ):
# make the source's context manager interface optional, call on source if there
if hasattr( self.source, '__exit__' ):
self.source.__exit__( *args )
# alternately, call close()
elif hasattr( self.source, 'close' ):
self.source.close()
def __str__( self ):
"""
String representation for easier debugging.
Will call `__str__` on it's source so this will display piped dataproviders.
"""
# we need to protect against recursion (in __getattr__) if self.source hasn't been set
source_str = str( self.source ) if hasattr( self, 'source' ) else ''
return '%s(%s)' %( self.__class__.__name__, str( source_str ) )
class FilteredDataProvider( DataProvider ):
"""
Passes each datum through a filter function and yields it if that function
returns a non-`None` value.
Also maintains counters:
- `num_data_read`: how many data have been consumed from the source.
- `num_valid_data_read`: how many data have been returned from `filter`.
- `num_data_returned`: how many data has this provider yielded.
"""
# not useful here - we don't want functions over the query string
#settings.update({ 'filter_fn': 'function' })
def __init__( self, source, filter_fn=None, **kwargs ):
"""
:param filter_fn: a lambda or function that will be passed a datum and
return either the (optionally modified) datum or None.
"""
super( FilteredDataProvider, self ).__init__( source, **kwargs )
self.filter_fn = filter_fn if hasattr( filter_fn, '__call__' ) else None
# count how many data we got from the source
self.num_data_read = 0
# how many valid data have we gotten from the source
# IOW, data that's passed the filter and been either provided OR have been skipped due to offset
self.num_valid_data_read = 0
# how many lines have been provided/output
self.num_data_returned = 0
def __iter__( self ):
parent_gen = super( FilteredDataProvider, self ).__iter__()
for datum in parent_gen:
self.num_data_read += 1
datum = self.filter( datum )
if datum != None:
self.num_valid_data_read += 1
self.num_data_returned += 1
yield datum
#TODO: may want to squash this into DataProvider
def filter( self, datum ):
"""
When given a datum from the provider's source, return None if the datum
'does not pass' the filter or is invalid. Return the datum if it's valid.
:param datum: the datum to check for validity.
:returns: the datum, a modified datum, or None
Meant to be overridden.
"""
if self.filter_fn:
return self.filter_fn( datum )
# also can be overriden entirely
return datum
class LimitedOffsetDataProvider( FilteredDataProvider ):
"""
A provider that uses the counters from FilteredDataProvider to limit the
number of data and/or skip `offset` number of data before providing.
Useful for grabbing sections from a source (e.g. pagination).
"""
# define the expected types of these __init__ arguments so they can be parsed out from query strings
settings = {
'limit' : 'int',
'offset': 'int'
}
#TODO: may want to squash this into DataProvider
def __init__( self, source, offset=0, limit=None, **kwargs ):
"""
:param offset: the number of data to skip before providing.
:param limit: the final number of data to provide.
"""
super( LimitedOffsetDataProvider, self ).__init__( source, **kwargs )
# how many valid data to skip before we start outputing data - must be positive
# (diff to support neg. indeces - must be pos.)
self.offset = max( offset, 0 )
# how many valid data to return - must be positive (None indicates no limit)
self.limit = limit
if self.limit != None:
self.limit = max( self.limit, 0 )
def __iter__( self ):
"""
Iterate over the source until `num_valid_data_read` is greater than
`offset`, begin providing datat, and stop when `num_data_returned`
is greater than `offset`.
"""
parent_gen = super( LimitedOffsetDataProvider, self ).__iter__()
for datum in parent_gen:
if self.limit != None and self.num_data_returned > self.limit:
break
if self.num_valid_data_read > self.offset:
yield datum
else:
# wot a cheezy way of doing this...
self.num_data_returned -= 1
#TODO: skipping lines is inefficient - somehow cache file position/line_num pair and allow provider
# to seek to a pos/line and then begin providing lines
# the important catch here is that we need to have accurate pos/line pairs
# in order to preserve the functionality of limit and offset
#if file_seek and len( file_seek ) == 2:
# seek_pos, new_line_num = file_seek
# self.seek_and_set_curr_line( seek_pos, new_line_num )
#def seek_and_set_curr_line( self, file_seek, new_curr_line_num ):
# self.seek( file_seek, os.SEEK_SET )
# self.curr_line_num = new_curr_line_num
class MultiSourceDataProvider( DataProvider ):
"""
A provider that iterates over a list of given sources and provides data
from one after another.
An iterator over iterators.
"""
def __init__( self, source_list, **kwargs ):
"""
:param source_list: an iterator of iterables
"""
self.source_list = deque( source_list )
def __iter__( self ):
"""
Iterate over the source_list, then iterate over the data in each source.
Skip a given source in `source_list` if it is `None` or invalid.
"""
for source in self.source_list:
# just skip falsy sources
if not source:
continue
try:
self.source = self.validate_source( source )
except exceptions.InvalidDataProviderSource, invalid_source:
continue
parent_gen = super( MultiSourceDataProvider, self ).__iter__()
for datum in parent_gen:
yield datum
lib/galaxy/datatypes/dataproviders/chunk.py
+84
View File
@@ -0,0 +1,84 @@
"""
Chunk (N number of bytes at M offset to a source's beginning) provider.
Primarily for file sources but usable by any iterator that has both
seek and read( N ).
"""
import os
import base64
import base
import exceptions
_TODO = """
"""
import logging
log = logging.getLogger( __name__ )
# -----------------------------------------------------------------------------
class ChunkDataProvider( base.DataProvider ):
"""
Data provider that yields chunks of data from it's file.
Note: this version does not account for lines and works with Binary datatypes.
"""
MAX_CHUNK_SIZE = 2**16
DEFAULT_CHUNK_SIZE = MAX_CHUNK_SIZE
settings = {
'chunk_index' : 'int',
'chunk_size' : 'int'
}
#TODO: subclass from LimitedOffsetDataProvider?
# see web/framework/base.iterate_file, util/__init__.file_reader, and datatypes.tabular
def __init__( self, source, chunk_index=0, chunk_size=DEFAULT_CHUNK_SIZE, **kwargs ):
"""
:param chunk_index: if a source can be divided into N number of
`chunk_size` sections, this is the index of which section to
return.
:param chunk_size: how large are the desired chunks to return
(gen. in bytes).
"""
super( ChunkDataProvider, self ).__init__( source, **kwargs )
self.chunk_size = int( chunk_size )
self.chunk_pos = int( chunk_index ) * self.chunk_size
def validate_source( self, source ):
"""
Does the given source have both the methods `seek` and `read`?
:raises InvalidDataProviderSource: if not.
"""
source = super( ChunkDataProvider, self ).validate_source( source )
if( ( not hasattr( source, 'seek' ) )
or ( not hasattr( source, 'read' ) ) ):
raise exceptions.InvalidDataProviderSource( source )
return source
def __iter__( self ):
# not reeeally an iterator per se
self.__enter__()
self.source.seek( self.chunk_pos, os.SEEK_SET )
chunk = self.encode( self.source.read( self.chunk_size ) )
yield chunk
self.__exit__()
def encode( self, chunk ):
"""
Called on the chunk before returning.
Overrride to modify, encode, or decode chunks.
"""
return chunk
class Base64ChunkDataProvider( ChunkDataProvider ):
"""
Data provider that yields chunks of base64 encoded data from it's file.
"""
def encode( self, chunk ):
"""
Return chunks encoded in base 64.
"""
return base64.b64encode( chunk )
lib/galaxy/datatypes/dataproviders/column.py
+254
View File
@@ -0,0 +1,254 @@
"""
Providers that provide lists of lists generally where each line of a source
is further subdivided into multiple data (e.g. columns from a line).
"""
import line
_TODO = """
move ColumnarDataProvider parsers to more sensible location
TransposedColumnarDataProvider: provides each column as a single array
- see existing visualizations/dataprovider/basic.ColumnDataProvider
"""
import logging
log = logging.getLogger( __name__ )
# ----------------------------------------------------------------------------- base classes
class ColumnarDataProvider( line.RegexLineDataProvider ):
"""
Data provider that provide a list of columns from the lines of it's source.
Columns are returned in the order given in indeces, so this provider can
re-arrange columns.
If any desired index is outside the actual number of columns
in the source, this provider will None-pad the output and you are guaranteed
the same number of columns as the number of indeces asked for (even if they
are filled with None).
"""
settings = {
'indeces' : 'list:int',
'column_count' : 'int',
'column_types' : 'list:str',
'parse_columns' : 'bool',
'deliminator' : 'str'
}
def __init__( self, source, indeces=None,
column_count=None, column_types=None, parsers=None, parse_columns=True,
deliminator='\t', **kwargs ):
"""
:param indeces: a list of indeces of columns to gather from each row
Optional: will default to `None`.
If `None`, this provider will return all rows (even when a
particular row contains more/less than others).
If a row/line does not contain an element at a given index, the
provider will-return/fill-with a `None` value as the element.
:type indeces: list or None
:param column_count: an alternate means of defining indeces, use an int
here to effectively provide the first N columns.
Optional: will default to `None`.
:type column_count: int
:param column_types: a list of string names of types that the
provider will use to look up an appropriate parser for the column.
(e.g. 'int', 'float', 'str', 'bool')
Optional: will default to parsing all columns as strings.
:type column_types: list of strings
:param parsers: a dictionary keyed with column type strings
and with values that are functions to use when parsing those
types.
Optional: will default to using the function `_get_default_parsers`.
:type parsers: dictionary
:param parse_columns: attempt to parse columns?
Optional: defaults to `True`.
:type parse_columns: bool
:param deliminator: character(s) used to split each row/line of the source.
Optional: defaults to the tab character.
:type deliminator: str
.. note: that the subclass constructors are passed kwargs - so they're
params (limit, offset, etc.) are also applicable here.
"""
#TODO: other columnar formats: csv, etc.
super( ColumnarDataProvider, self ).__init__( source, **kwargs )
#IMPLICIT: if no indeces, column_count, or column_types passed: return all columns
self.selected_column_indeces = indeces
self.column_count = column_count
self.column_types = column_types or []
# if no column count given, try to infer from indeces or column_types
if not self.column_count:
if self.selected_column_indeces:
self.column_count = len( self.selected_column_indeces )
elif self.column_types:
self.column_count = len( self.column_types )
# if no indeces given, infer from column_count
if not self.selected_column_indeces and self.column_count:
self.selected_column_indeces = list( xrange( self.column_count ) )
self.deliminator = deliminator
# how/whether to parse each column value
self.parsers = {}
if parse_columns:
self.parsers = self.get_default_parsers()
# overwrite with user desired parsers
self.parsers.update( parsers or {} )
def get_default_parsers( self ):
"""
Return parser dictionary keyed for each columnar type
(as defined in datatypes).
.. note: primitives only by default (str, int, float, boolean, None).
Other (more complex) types are retrieved as strings.
:returns: a dictionary of the form:
`{ <parser type name> : <function used to parse type> }`
"""
#TODO: move to module level (or datatypes, util)
return {
# str is default and not needed here
'int' : int,
'float' : float,
'bool' : bool,
# unfortunately, 'list' is used in dataset metadata both for
# query style maps (9th col gff) AND comma-sep strings.
# (disabled for now)
#'list' : lambda v: v.split( ',' ),
#'csv' : lambda v: v.split( ',' ),
## i don't like how urlparses does sub-lists...
#'querystr' : lambda v: dict([ ( p.split( '=', 1 ) if '=' in p else ( p, True ) )
# for p in v.split( ';', 1 ) ])
#'scifloat': #floating point which may be in scientific notation
# always with the 1 base, biologists?
#'int1' : ( lambda i: int( i ) - 1 ),
#'gffval': string or '.' for None
#'gffint': # int or '.' for None
#'gffphase': # 0, 1, 2, or '.' for None
#'gffstrand': # -, +, ?, or '.' for None, etc.
}
def parse_value( self, val, type ):
"""
Attempt to parse and return the given value based on the given type.
:param val: the column value to parse (often a string)
:param type: the string type 'name' used to find the appropriate parser
:returns: the parsed value
or `value` if no `type` found in `parsers`
or `None` if there was a parser error (ValueError)
"""
if type == 'str' or type == None: return val
try:
return self.parsers[ type ]( val )
except KeyError, err:
# no parser - return as string
pass
except ValueError, err:
# bad value - return None
return None
return val
def get_column_type( self, index ):
"""
Get the column type for the parser from `self.column_types` or `None`
if the type is unavailable.
:param index: the column index
:returns: string name of type (e.g. 'float', 'int', etc.)
"""
try:
return self.column_types[ index ]
except IndexError, ind_err:
return None
def parse_column_at_index( self, columns, parser_index, index ):
"""
Get the column type for the parser from `self.column_types` or `None`
if the type is unavailable.
"""
try:
return self.parse_value( columns[ index ], self.get_column_type( parser_index ) )
# if a selected index is not within columns, return None
except IndexError, index_err:
return None
def parse_columns_from_line( self, line ):
"""
Returns a list of the desired, parsed columns.
:param line: the line to parse
:type line: str
"""
#TODO: too much going on in this loop - the above should all be precomputed AMAP...
all_columns = line.split( self.deliminator )
# if no indeces were passed to init, return all columns
selected_indeces = self.selected_column_indeces or list( xrange( len( all_columns ) ) )
parsed_columns = []
for parser_index, column_index in enumerate( selected_indeces ):
parsed_columns.append( self.parse_column_at_index( all_columns, parser_index, column_index ) )
return parsed_columns
def __iter__( self ):
parent_gen = super( ColumnarDataProvider, self ).__iter__()
for line in parent_gen:
columns = self.parse_columns_from_line( line )
yield columns
#TODO: implement column filters here and not below - flatten hierarchy
class FilteredByColumnDataProvider( ColumnarDataProvider ):
"""
Data provider that provide a list of columns from the lines of it's source
_only_ if they pass a given filter function.
e.g. column #3 is type int and > N
"""
# TODO: how to do this and still have limit and offset work?
def __init__( self, source, **kwargs ):
raise NotImplementedError()
super( FilteredByColumnDataProvider, self ).__init__( source, **kwargs )
class DictDataProvider( ColumnarDataProvider ):
"""
Data provider that zips column_names and columns from the source's contents
into a dictionary.
A combination use of both `column_names` and `indeces` allows 'picking'
key/value pairs from the source.
.. note: that the subclass constructors are passed kwargs - so they're
params (limit, offset, etc.) are also applicable here.
"""
settings = {
'column_names' : 'list:str',
}
def __init__( self, source, column_names=None, **kwargs ):
"""
:param column_names: an ordered list of strings that will be used as the keys
for each column in the returned dictionaries.
The number of key, value pairs each returned dictionary has will
be as short as the number of column names provided.
:type column_names:
"""
#TODO: allow passing in a map instead of name->index { 'name1': index1, ... }
super( DictDataProvider, self ).__init__( source, **kwargs )
self.column_names = column_names or []
def __iter__( self ):
parent_gen = super( DictDataProvider, self ).__iter__()
for column_values in parent_gen:
map = dict( zip( self.column_names, column_values ) )
yield map
lib/galaxy/datatypes/dataproviders/dataset.py
+716
View File
@@ -0,0 +1,716 @@
"""
Dataproviders that use either:
- the file contents and/or metadata from a Galaxy DatasetInstance as
their source.
- or provide data in some way relevant to bioinformatic data
(e.g. parsing genomic regions from their source)
"""
import pkg_resources
pkg_resources.require( 'bx-python' )
from bx import seq as bx_seq
from bx import wiggle as bx_wig
import galaxy.model
import galaxy.datatypes
import galaxy.datatypes.data
#TODO: can't import these due to circular ref in model/registry
#import galaxy.datatypes.binary
#import galaxy.datatypes.tabular
import exceptions
import base
import line
import column
import external
_TODO = """
use bx as much as possible
the use of DatasetInstance seems to create some import problems
gff3 hierarchies
"""
import logging
log = logging.getLogger( __name__ )
# ----------------------------------------------------------------------------- base for using a Glx dataset
class DatasetDataProvider( base.DataProvider ):
"""
Class that uses the file contents and/or metadata from a Galaxy DatasetInstance
as it's source.
DatasetDataProvider can be seen as the intersection between a datatype's
metadata and a dataset's file contents. It (so far) mainly provides helper
and conv. methods for using dataset metadata to set up and control how
the data is provided.
"""
def __init__( self, dataset, **kwargs ):
"""
:param dataset: the Galaxy dataset whose file will be the source
:type dataset: model.DatasetInstance
:raises exceptions.InvalidDataProviderSource: if not a DatsetInstance
"""
if not isinstance( dataset, galaxy.model.DatasetInstance ):
raise exceptions.InvalidDataProviderSource( "Data provider can only be used with a DatasetInstance" )
self.dataset = dataset
# this dataset file is obviously the source
#TODO: this might be a good place to interface with the object_store...
super( DatasetDataProvider, self ).__init__( open( dataset.file_name, 'rb' ) )
#TODO: this is a bit of a mess
@classmethod
def get_column_metadata_from_dataset( cls, dataset ):
"""
Convenience class method to get column metadata from a dataset.
:returns: a dictionary of `column_count`, `column_types`, and `column_names`
if they're available, setting each to `None` if not.
"""
# re-map keys to fit ColumnarProvider.__init__ kwargs
params = {}
params[ 'column_count' ] = dataset.metadata.columns
params[ 'column_types' ] = dataset.metadata.column_types
params[ 'column_names' ] = dataset.metadata.column_names or getattr( dataset.datatype, 'column_names', None )
return params
def get_metadata_column_types( self, indeces=None ):
"""
Return the list of `column_types` for this dataset or `None` if unavailable.
:param indeces: the indeces for the columns of which to return the types.
Optional: defaults to None (return all types)
:type indeces: list of ints
"""
metadata_column_types = ( self.dataset.metadata.column_types
or getattr( self.dataset.datatype, 'column_types', None )
or None )
if not metadata_column_types:
return metadata_column_types
if indeces:
column_types = []
for index in indeces:
column_type = metadata_column_types[ index ] if index < len( metadata_column_types ) else None
column_types.append( column_type )
return column_types
return metadata_column_types
def get_metadata_column_names( self, indeces=None ):
"""
Return the list of `column_names` for this dataset or `None` if unavailable.
:param indeces: the indeces for the columns of which to return the names.
Optional: defaults to None (return all names)
:type indeces: list of ints
"""
metadata_column_names = ( self.dataset.metadata.column_names
or getattr( self.dataset.datatype, 'column_names', None )
or None )
if not metadata_column_names:
return metadata_column_names
if indeces:
column_names = []
for index in indeces:
column_type = metadata_column_names[ index ] if index < len( metadata_column_names ) else None
column_names.append( column_type )
return column_names
return metadata_column_names
#TODO: merge the next two
def get_indeces_by_column_names( self, list_of_column_names ):
"""
Return the list of column indeces when given a list of column_names.
:param list_of_column_names: the names of the columns of which to get indeces.
:type list_of_column_names: list of strs
:raises KeyError: if column_names are not found
:raises ValueError: if an entry in list_of_column_names is not in column_names
"""
metadata_column_names = ( self.dataset.metadata.column_names
or getattr( self.dataset.datatype, 'column_names', None )
or None )
if not metadata_column_names:
raise KeyError( 'No column_names found for '
+ 'datatype: %s, dataset: %s' %( str( self.dataset.datatype ), str( self.dataset ) ) )
indeces = []
for column_name in list_of_column_names:
indeces.append( metadata_column_names.index( column_name ) )
return indeces
def get_metadata_column_index_by_name( self, name ):
"""
Return the 1-base index of a sources column with the given `name`.
"""
# metadata columns are 1-based indeces
column = getattr( self.dataset.metadata, name )
return ( column - 1 ) if ( isinstance( column, int ) and column > 0 ) else None
def get_genomic_region_indeces( self, check=False ):
"""
Return a list of column indeces for 'chromCol', 'startCol', 'endCol' from
a source representing a genomic region.
:param check: if True will raise a ValueError if any were not found.
:type check: bool
:raises ValueError: if check is `True` and one or more indeces were not found.
:returns: list of column indeces for the named columns.
"""
region_column_names = ( 'chromCol', 'startCol', 'endCol' )
region_indeces = [ self.get_metadata_column_index_by_name( name ) for name in region_column_names ]
if check and not all( map( lambda i: i != None, indeces ) ):
raise ValueError( "Could not determine proper column indeces for chrom, start, end: %s" %( str( indeces ) ) )
return region_indeces
class ConvertedDatasetDataProvider( DatasetDataProvider ):
"""
Class that uses the file contents of a dataset after conversion to a different
format.
"""
def __init__( self, dataset, **kwargs ):
raise NotImplementedError( 'Abstract class' )
self.original_dataset = dataset
self.converted_dataset = self.convert_dataset( dataset, **kwargs )
super( ConvertedDatasetDataProvider, self ).__init__( self.converted_dataset, **kwargs )
#NOTE: now self.converted_dataset == self.dataset
def convert_dataset( self, dataset, **kwargs ):
"""
Convert the given dataset in some way.
"""
return dataset
# ----------------------------------------------------------------------------- uses metadata for settings
class DatasetColumnarDataProvider( column.ColumnarDataProvider ):
"""
Data provider that uses a DatasetDataProvider as it's source and the
dataset's metadata to buuild settings for the ColumnarDataProvider it's
inherited from.
"""
def __init__( self, dataset, **kwargs ):
"""
All kwargs are inherited from ColumnarDataProvider.
.. seealso:: column.ColumnarDataProvider
If no kwargs are given, this class will attempt to get those kwargs
from the dataset source's metadata.
If any kwarg is given, it will override and be used in place of
any metadata available.
"""
dataset_source = DatasetDataProvider( dataset )
if not kwargs.get( 'column_types', None ):
indeces = kwargs.get( 'indeces', None )
kwargs[ 'column_types' ] = dataset_source.get_metadata_column_types( indeces=indeces )
super( DatasetColumnarDataProvider, self ).__init__( dataset_source, **kwargs )
class DatasetDictDataProvider( column.DictDataProvider ):
"""
Data provider that uses a DatasetDataProvider as it's source and the
dataset's metadata to buuild settings for the DictDataProvider it's
inherited from.
"""
def __init__( self, dataset, **kwargs ):
"""
All kwargs are inherited from DictDataProvider.
.. seealso:: column.DictDataProvider
If no kwargs are given, this class will attempt to get those kwargs
from the dataset source's metadata.
If any kwarg is given, it will override and be used in place of
any metadata available.
The relationship between column_names and indeces is more complex:
+-----------------+-------------------------------+-----------------------+
| | Indeces given | Indeces NOT given |
+=================+===============================+=======================+
| Names given | pull indeces, rename w/ names | pull by name |
+=================+-------------------------------+-----------------------+
| Names NOT given | pull indeces, name w/ meta | pull all, name w/meta |
+=================+-------------------------------+-----------------------+
"""
dataset_source = DatasetDataProvider( dataset )
#TODO: getting too complicated - simplify at some lvl, somehow
# if no column_types given, get column_types from indeces (or all if indeces == None)
indeces = kwargs.get( 'indeces', None )
column_names = kwargs.get( 'column_names', None )
#if indeces and column_names:
# # pull using indeces and re-name with given names - no need to alter (does as super would)
# pass
if not indeces and column_names:
# pull columns by name
indeces = kwargs[ 'indeces' ] = dataset_source.get_indeces_by_column_names( column_names )
elif indeces and not column_names:
# pull using indeces, name with meta
column_names = kwargs[ 'column_names' ] = dataset_source.get_metadata_column_names( indeces=indeces )
elif not indeces and not column_names:
# pull all indeces and name using metadata
column_names = kwargs[ 'column_names' ] = dataset_source.get_metadata_column_names( indeces=indeces )
# if no column_types given, use metadata column_types
if not kwargs.get( 'column_types', None ):
kwargs[ 'column_types' ] = dataset_source.get_metadata_column_types( indeces=indeces )
super( DatasetDictDataProvider, self ).__init__( dataset_source, **kwargs )
# ----------------------------------------------------------------------------- provides a bio-relevant datum
class GenomicRegionDataProvider( column.ColumnarDataProvider ):
"""
Data provider that parses chromosome, start, and end data from a file
using the datasets metadata settings.
Is a ColumnarDataProvider that uses a DatasetDataProvider as it's source.
If `named_columns` is true, will return dictionaries with the keys
'chrom', 'start', 'end'.
"""
# dictionary keys when named_columns=True
COLUMN_NAMES = [ 'chrom', 'start', 'end' ]
settings = {
'chrom_column' : 'int',
'start_column' : 'int',
'end_column' : 'int',
'named_columns' : 'bool',
}
def __init__( self, dataset, chrom_column=None, start_column=None, end_column=None, named_columns=False, **kwargs ):
"""
:param dataset: the Galaxy dataset whose file will be the source
:type dataset: model.DatasetInstance
:param chrom_column: optionally specify the chrom column index
:type chrom_column: int
:param start_column: optionally specify the start column index
:type start_column: int
:param end_column: optionally specify the end column index
:type end_column: int
:param named_columns: optionally return dictionaries keying each column
with 'chrom', 'start', or 'end'.
Optional: defaults to False
:type named_columns: bool
"""
#TODO: allow passing in a string format e.g. "{chrom}:{start}-{end}"
dataset_source = DatasetDataProvider( dataset )
if chrom_column == None:
chrom_column = dataset_source.get_metadata_column_index_by_name( 'chromCol' )
if start_column == None:
start_column = dataset_source.get_metadata_column_index_by_name( 'startCol' )
if end_column == None:
end_column = dataset_source.get_metadata_column_index_by_name( 'endCol' )
indeces = [ chrom_column, start_column, end_column ]
if not all( map( lambda i: i != None, indeces ) ):
raise ValueError( "Could not determine proper column indeces for"
+ " chrom, start, end: %s" %( str( indeces ) ) )
kwargs.update({ 'indeces' : indeces })
if not kwargs.get( 'column_types', None ):
kwargs.update({ 'column_types' : dataset_source.get_metadata_column_types( indeces=indeces ) })
self.named_columns = named_columns
if self.named_columns:
self.column_names = self.COLUMN_NAMES
super( GenomicRegionDataProvider, self ).__init__( dataset_source, **kwargs )
def __iter__( self ):
parent_gen = super( GenomicRegionDataProvider, self ).__iter__()
for column_values in parent_gen:
if self.named_columns:
yield dict( zip( self.column_names, column_values ) )
else:
yield column_values
#TODO: this optionally provides the same data as the above and makes GenomicRegionDataProvider redundant
# GenomicRegionDataProvider is a better name, tho
class IntervalDataProvider( column.ColumnarDataProvider ):
"""
Data provider that parses chromosome, start, and end data (as well as strand
and name if set in the metadata) using the dataset's metadata settings.
If `named_columns` is true, will return dictionaries with the keys
'chrom', 'start', 'end' (and 'strand' and 'name' if available).
"""
COLUMN_NAMES = [ 'chrom', 'start', 'end', 'strand', 'name' ]
settings = {
'chrom_column' : 'int',
'start_column' : 'int',
'end_column' : 'int',
'strand_column' : 'int',
'name_column' : 'int',
'named_columns' : 'bool',
}
def __init__( self, dataset, chrom_column=None, start_column=None, end_column=None,
strand_column=None, name_column=None, named_columns=False, **kwargs ):
"""
:param dataset: the Galaxy dataset whose file will be the source
:type dataset: model.DatasetInstance
:param named_columns: optionally return dictionaries keying each column
with 'chrom', 'start', 'end', 'strand', or 'name'.
Optional: defaults to False
:type named_columns: bool
"""
#TODO: allow passing in a string format e.g. "{chrom}:{start}-{end}"
dataset_source = DatasetDataProvider( dataset )
# get genomic indeces and add strand and name
self.column_names = []
indeces = []
#TODO: this is sort of involved and oogly
if chrom_column == None:
chrom_column = dataset_source.get_metadata_column_index_by_name( 'chromCol' )
if chrom_column != None:
self.column_names.append( 'chrom' )
indeces.append( chrom_column )
if start_column == None:
start_column = dataset_source.get_metadata_column_index_by_name( 'startCol' )
if start_column != None:
self.column_names.append( 'start' )
indeces.append( start_column )
if end_column == None:
end_column = dataset_source.get_metadata_column_index_by_name( 'endCol' )
if end_column != None:
self.column_names.append( 'end' )
indeces.append( end_column )
if strand_column == None:
strand_column = dataset_source.get_metadata_column_index_by_name( 'strandCol' )
if strand_column != None:
self.column_names.append( 'strand' )
indeces.append( strand_column )
if name_column == None:
name_column = dataset_source.get_metadata_column_index_by_name( 'nameCol' )
if name_column != None:
self.column_names.append( 'name' )
indeces.append( name_column )
kwargs.update({ 'indeces' : indeces })
if not kwargs.get( 'column_types', None ):
kwargs.update({ 'column_types' : dataset_source.get_metadata_column_types( indeces=indeces ) })
self.named_columns = named_columns
super( IntervalDataProvider, self ).__init__( dataset_source, **kwargs )
def __iter__( self ):
parent_gen = super( IntervalDataProvider, self ).__iter__()
for column_values in parent_gen:
if self.named_columns:
yield dict( zip( self.column_names, column_values ) )
else:
yield column_values
#TODO: ideally with these next two - you'd allow pulling some region from the sequence
# WITHOUT reading the entire seq into memory - possibly apply some version of limit/offset
class FastaDataProvider( base.FilteredDataProvider ):
"""
Class that returns fasta format data in a list of maps of the form:
{
id: <fasta header id>,
sequence: <joined lines of nucleotide/amino data>
}
"""
settings = {
'ids' : 'list:str',
}
def __init__( self, source, ids=None, **kwargs ):
"""
:param ids: optionally return only ids (and sequences) that are in this list.
Optional: defaults to None (provide all ids)
:type ids: list or None
"""
source = bx_seq.fasta.FastaReader( source )
#TODO: validate is a fasta
super( FastaDataProvider, self ).__init__( source, **kwargs )
self.ids = ids
# how to do ids?
def __iter__( self ):
parent_gen = super( FastaDataProvider, self ).__iter__()
for fasta_record in parent_gen:
yield {
'id' : fasta_record.name,
'seq' : fasta_record.text
}
class TwoBitFastaDataProvider( DatasetDataProvider ):
"""
Class that returns fasta format data in a list of maps of the form:
{
id: <fasta header id>,
sequence: <joined lines of nucleotide/amino data>
}
"""
settings = {
'ids' : 'list:str',
}
def __init__( self, source, ids=None, **kwargs ):
"""
:param ids: optionally return only ids (and sequences) that are in this list.
Optional: defaults to None (provide all ids)
:type ids: list or None
"""
source = bx_seq.twobit.TwoBitFile( source )
#TODO: validate is a 2bit
super( FastaDataProvider, self ).__init__( source, **kwargs )
# could do in order provided with twobit
self.ids = ids or self.source.keys()
def __iter__( self ):
for id_ in self.ids:
yield {
'id' : id_,
'seq' : self.source[ name ]
}
#TODO:
class WiggleDataProvider( base.LimitedOffsetDataProvider ):
"""
Class that returns chrom, pos, data from a wiggle source.
"""
COLUMN_NAMES = [ 'chrom', 'pos', 'value' ]
settings = {
'named_columns' : 'bool',
'column_names' : 'list:str',
}
def __init__( self, source, named_columns=False, column_names=None, **kwargs ):
"""
:param named_columns: optionally return dictionaries keying each column
with 'chrom', 'start', 'end', 'strand', or 'name'.
Optional: defaults to False
:type named_columns: bool
:param column_names: an ordered list of strings that will be used as the keys
for each column in the returned dictionaries.
The number of key, value pairs each returned dictionary has will
be as short as the number of column names provided.
:type column_names:
"""
#TODO: validate is a wig
# still good to maintain a ref to the raw source bc Reader won't
self.raw_source = source
self.parser = bx_wig.Reader( source )
super( WiggleDataProvider, self ).__init__( self.parser, **kwargs )
self.named_columns = named_columns
self.column_names = column_names or self.COLUMN_NAMES
def __iter__( self ):
parent_gen = super( WiggleDataProvider, self ).__iter__()
for three_tuple in parent_gen:
if self.named_columns:
yield dict( zip( self.column_names, three_tuple ) )
else:
# list is not strictly necessary - but consistent
yield list( three_tuple )
class BigWigDataProvider( base.LimitedOffsetDataProvider ):
"""
Class that returns chrom, pos, data from a wiggle source.
"""
COLUMN_NAMES = [ 'chrom', 'pos', 'value' ]
settings = {
'named_columns' : 'bool',
'column_names' : 'list:str',
}
def __init__( self, source, chrom, start, end, named_columns=False, column_names=None, **kwargs ):
"""
:param chrom: which chromosome within the bigbed file to extract data for
:type chrom: str
:param start: the start of the region from which to extract data
:type start: int
:param end: the end of the region from which to extract data
:type end: int
:param named_columns: optionally return dictionaries keying each column
with 'chrom', 'start', 'end', 'strand', or 'name'.
Optional: defaults to False
:type named_columns: bool
:param column_names: an ordered list of strings that will be used as the keys
for each column in the returned dictionaries.
The number of key, value pairs each returned dictionary has will
be as short as the number of column names provided.
:type column_names:
"""
raise NotImplementedError( 'Work in progress' )
#TODO: validate is a wig
# still good to maintain a ref to the raw source bc Reader won't
self.raw_source = source
self.parser = bx_bbi.bigwig_file.BigWigFile( source )
super( BigWigDataProvider, self ).__init__( self.parser, **kwargs )
self.named_columns = named_columns
self.column_names = column_names or self.COLUMN_NAMES
def __iter__( self ):
parent_gen = super( BigWigDataProvider, self ).__iter__()
for three_tuple in parent_gen:
if self.named_columns:
yield dict( zip( self.column_names, three_tuple ) )
else:
# list is not strictly necessary - but consistent
yield list( three_tuple )
# ----------------------------------------------------------------------------- binary, external conversion or tool
class DatasetSubprocessDataProvider( external.SubprocessDataProvider ):
"""
Create a source from running a subprocess on a dataset's file.
Uses a subprocess as it's source and has a dataset (gen. as an input file
for the process).
"""
#TODO: below should be a subclass of this and not RegexSubprocess
def __init__( self, dataset, *args, **kwargs ):
"""
:param args: the list of strings used to build commands.
:type args: variadic function args
"""
raise NotImplementedError( 'Abstract class' )
super( DatasetSubprocessDataProvider, self ).__init__( *args, **kwargs )
self.dataset = dataset
class SamtoolsDataProvider( line.RegexLineDataProvider ):
"""
Data provider that uses samtools on a Sam or Bam file as it's source.
This can be piped through other providers (column, map, genome region, etc.).
.. note:: that only the samtools 'view' command is currently implemented.
"""
FLAGS_WO_ARGS = 'bhHSu1xXcB'
FLAGS_W_ARGS = 'fFqlrs'
VALID_FLAGS = FLAGS_WO_ARGS + FLAGS_W_ARGS
def __init__( self, dataset, options_string='', options_dict=None, regions=None, **kwargs ):
"""
:param options_string: samtools options in string form (flags separated
by spaces)
Optional: defaults to ''
:type options_string: str
:param options_dict: dictionary of samtools options
Optional: defaults to None
:type options_dict: dict or None
:param regions: list of samtools regions strings
Optional: defaults to None
:type regions: list of str or None
"""
#TODO: into validate_source
#TODO: have to import these here due to circular ref in model/datatypes
import galaxy.datatypes.binary
import galaxy.datatypes.tabular
if( not( isinstance( dataset.datatype, galaxy.datatypes.tabular.Sam )
or isinstance( dataset.datatype, galaxy.datatypes.binary.Bam ) ) ):
raise exceptions.InvalidDataProviderSource(
'dataset must be a Sam or Bam datatype: %s' %( str( dataset.datatype ) ) )
self.dataset = dataset
options_dict = options_dict or {}
# ensure regions are strings
regions = [ str( r ) for r in regions ] if regions else []
#TODO: view only for now
#TODO: not properly using overriding super's validate_opts, command here
subcommand = 'view'
#TODO:?? do we need a path to samtools?
subproc_args = self.build_command_list( subcommand, options_string, options_dict, regions )
#TODO: the composition/inheritance here doesn't make a lot sense
subproc_provider = external.SubprocessDataProvider( *subproc_args )
super( SamtoolsDataProvider, self ).__init__( subproc_provider, **kwargs )
def build_command_list( self, subcommand, options_string, options_dict, regions ):
"""
Convert all init args to list form.
"""
command = [ 'samtools', subcommand ]
# add options and switches, input file, regions list (if any)
command.extend( self.to_options_list( options_string, options_dict ) )
command.append( self.dataset.file_name )
command.extend( regions )
return command
def to_options_list( self, options_string, options_dict ):
"""
Convert both options_string and options_dict to list form
while filtering out non-'valid' options.
"""
opt_list = []
# strip out any user supplied bash switch formating -> string of option chars
# then compress to single option string of unique, VALID flags with prefixed bash switch char '-'
options_string = options_string.strip( '- ' )
validated_flag_list = set([ flag for flag in options_string if flag in self.FLAGS_WO_ARGS ])
# if sam add -S
if( ( isinstance( self.dataset.datatype, galaxy.datatypes.tabular.Sam )
and ( 'S' not in validated_flag_list ) ) ):
validated_flag_list.append( 'S' )
if validated_flag_list:
opt_list.append( '-' + ''.join( validated_flag_list ) )
for flag, arg in options_dict.items():
if flag in self.FLAGS_W_ARGS:
opt_list.extend([ '-' + flag, str( arg ) ])
return opt_list
@classmethod
def extract_options_from_dict( cls, dictionary ):
"""
Separrates valid samtools key/value pair options from a dictionary and
returns both as a 2-tuple.
"""
# handy for extracting options from kwargs - but otherwise...
#TODO: could be abstracted to util.extract( dict, valid_keys_list )
options_dict = {}
new_kwargs = {}
for key, value in dictionary.items():
if key in cls.FLAGS_W_ARGS:
options_dict[ key ] = value
else:
new_kwargs[ key ] = value
return options_dict, new_kwargs
class BcftoolsDataProvider( line.RegexLineDataProvider ):
"""
Data provider that uses an bcftools on a bcf (or vcf?) file as it's source.
This can be piped through other providers (column, map, genome region, etc.).
"""
def __init__( self, dataset, **kwargs ):
#TODO: as samtools
raise NotImplementedError()
super( BCFDataProvider, self ).__init__( dataset, **kwargs )
class BGzipTabixDataProvider( base.DataProvider ):
"""
Data provider that uses an g(un)zip on a file as it's source.
This can be piped through other providers (column, map, genome region, etc.).
"""
def __init__( self, dataset, **kwargs ):
#TODO: as samtools - need more info on output format
raise NotImplementedError()
super( BGzipTabixDataProvider, self ).__init__( dataset, **kwargs )
lib/galaxy/datatypes/dataproviders/decorators.py
+165
View File
@@ -0,0 +1,165 @@
"""
DataProvider related decorators.
"""
# I'd like to decorate the factory methods that give data_providers by the name they can be accessed from. e.g.:
#@provides( 'id_seq' ) # where 'id_seq' is some 'data_format' string/alias
#def get_id_seq_provider( dataset, **settings ):
# then in some central dispatch (maybe data.Data), have it look up the proper method by the data_format string
# also it would be good to have this decorator maintain a list of available providers (for a datatype)
# i don't particularly want to cut up method names ( get_([\w_]*)_provider )
#!/usr/bin/env python
# adapted from: http://stackoverflow.com
# /questions/14095616/python-can-i-programmatically-decorate-class-methods-from-a-class-instance
from functools import wraps
#from types import MethodType
import copy
import logging
log = logging.getLogger( __name__ )
# -----------------------------------------------------------------------------
_DATAPROVIDER_CLASS_MAP_KEY = 'dataproviders'
_DATAPROVIDER_METHOD_NAME_KEY = '_dataprovider_name'
# -----------------------------------------------------------------------------
def has_dataproviders( cls ):
"""
Wraps a class (generally a Datatype), finds methods within that have been
decorated with `@dataprovider` and adds them, by their name, to a map
in the class.
This allows a class to maintain a name -> method map, effectively
'registering' dataprovider factory methods.
.. example::
@has_dataproviders
class MyDtype( data.Data ):
@dataprovider_factory( 'bler' )
def provide_some_bler( self, dataset, **settings ):
'''blerblerbler'''
dataset_source = providers.DatasetDataProvider( dataset )
# ... chain other, intermidiate providers here
return providers.BlerDataProvider( dataset_source, **settings )
# use the base method in data.Data
provider = dataset.datatype.dataprovider( dataset, 'bler',
my_setting='blah', ... )
# OR directly from the map
provider = dataset.datatype.dataproviders[ 'bler' ]( dataset,
my_setting='blah', ... )
"""
#log.debug( 'has_dataproviders:', cls )
# init the class dataproviders map if necc.
if not hasattr( cls, _DATAPROVIDER_CLASS_MAP_KEY ):
setattr( cls, _DATAPROVIDER_CLASS_MAP_KEY, {} )
else:
# need to deepcopy or subclasses will modify super.dataproviders as well
existing_dataproviders = getattr( cls, _DATAPROVIDER_CLASS_MAP_KEY )
copied_dataproviders = copy.deepcopy( existing_dataproviders )
setattr( cls, _DATAPROVIDER_CLASS_MAP_KEY, copied_dataproviders )
dataproviders = getattr( cls, _DATAPROVIDER_CLASS_MAP_KEY )
# scan for methods with dataprovider names and add them to the map
# note: this has a 'cascading' effect
# where it's possible to override a super's provider with a sub's
for attr_key, attr_value in cls.__dict__.iteritems():
#log.debug( '\t key:', attr_key )
# can't use isinstance( attr_value, MethodType ) bc of wrapping
if( ( callable( attr_value ) )
and ( not attr_key.startswith( "__" ) )
and ( getattr( attr_value, _DATAPROVIDER_METHOD_NAME_KEY, None ) ) ):
#log.debug( '\t\t is a dataprovider', attr_key )
name = getattr( attr_value, _DATAPROVIDER_METHOD_NAME_KEY )
dataproviders[ name ] = attr_value
#log.debug( 'dataproviders:' )
#for name, fn in cls.dataproviders.items():
# log.debug( '\t ', name, '->', fn.__name__, fn )
# log.debug( '\t\t ', fn.__doc__ )
return cls
def dataprovider_factory( name, settings=None ):
"""
Wraps a class method and marks it as a dataprovider factory and creates a
function to parse query strings to __init__ arguments as the
`parse_query_string_settings` attribute of the factory function.
An example use of the `parse_query_string_settings`:
..example::
kwargs = dataset.datatype.dataproviders[ provider ].parse_query_string_settings( query_kwargs )
return list( dataset.datatype.dataprovider( dataset, provider, **kwargs ) )
:param name: what name/key to register the factory under in `cls.dataproviders`
:type name: any hashable var
:param settings: dictionary containing key/type pairs for parsing query strings
to __init__ arguments
:type settings: dictionary
"""
#TODO:?? use *args for settings allowing mulitple dictionaries
# make a function available through the name->provider dispatch to parse query strings
# callable like:
# settings_dict = dataproviders[ provider_name ].parse_query_string_settings( query_kwargs )
#TODO: ugh - overly complicated but the best I could think of
def parse_query_string_settings( query_kwargs ):
return _parse_query_string_settings( query_kwargs, settings )
#log.debug( 'dataprovider:', name )
def named_dataprovider_factory( func ):
#log.debug( 'named_dataprovider_factory:', name, '->', func.__name__ )
setattr( func, _DATAPROVIDER_METHOD_NAME_KEY, name )
setattr( func, 'parse_query_string_settings', parse_query_string_settings )
setattr( func, 'settings', settings )
#TODO: I want a way to inherit settings from the previous provider( this_name ) instead of defining over and over
#log.debug( '\t setting:', getattr( func, _DATAPROVIDER_METHOD_NAME_KEY ) )
@wraps( func )
def wrapped_dataprovider_factory( self, *args, **kwargs ):
#log.debug( 'wrapped_dataprovider_factory', name, self, args, kwargs )
return func( self, *args, **kwargs )
return wrapped_dataprovider_factory
return named_dataprovider_factory
def _parse_query_string_settings( query_kwargs, settings=None ):
"""
Parse the values in `query_kwargs` from strings to the proper types
listed in the same key in `settings`.
"""
def list_from_query_string( s ):
# assume csv
return s.split( ',' )
parsers = {
'int' : int,
'float' : float,
'bool' : bool,
'list:str' : lambda s: list_from_query_string( s ),
'list:int' : lambda s: [ int( i ) for i in list_from_query_string( s ) ],
}
settings = settings or {}
# yay! yet another set of query string parsers! <-- sarcasm
# work through the keys in settings finding matching keys in query_kwargs
# if found in both, get the expected/needed type from settings and store the new parsed value
# if we can't parse it (no parser, bad value), delete the key from query_kwargs so the provider will use the defaults
for key in settings:
if key in query_kwargs:
#TODO: this would be the place to sanitize any strings
query_value = query_kwargs[ key ]
needed_type = settings[ key ]
try:
query_kwargs[ key ] = parsers[ needed_type ]( query_value )
except ( KeyError, ValueError ):
del query_kwargs[ key ]
#TODO:?? do we want to remove query_kwarg entries NOT in settings?
return query_kwargs
lib/galaxy/datatypes/dataproviders/exceptions.py
+33
View File
@@ -0,0 +1,33 @@
"""
DataProvider related exceptions.
"""
class InvalidDataProviderSource( TypeError ):
"""
Raised when a unusable source is passed to a provider.
"""
def __init__( self, source=None, msg='' ):
msg = msg or 'Invalid source for provider: %s' %( source )
super( InvalidDataProviderSource, self ).__init__( msg )
class NoProviderAvailable( TypeError ):
"""
Raised when no provider is found for the given `format_requested`.
:param factory_source: the item that the provider was requested from
:param format_requested: the format_requested (a hashable key to access
`factory_source.datatypes` with)
Both params are attached to this class and accessible to the try-catch
receiver.
Meant to be used within a class that builds dataproviders (e.g. a Datatype)
"""
def __init__( self, factory_source, format_requested=None, msg='' ):
self.factory_source = factory_source
self.format_requested = format_requested
msg = msg or 'No provider available in factory_source "%s" for format requested' %( str( factory_source ) )
if self.format_requested:
msg += ': "%s"' %( self.format_requested )
super( NoProviderAvailable, self ).__init__( msg )
lib/galaxy/datatypes/dataproviders/external.py
+165
View File
@@ -0,0 +1,165 @@
"""
Data providers that iterate over a source that is not in memory
or not in a file.
"""
import subprocess
import urllib, urllib2
import gzip
import base
import line
_TODO = """
YAGNI: ftp, image, cryptos, sockets
job queue
admin: admin server log rgx/stats, ps aux
"""
import logging
log = logging.getLogger( __name__ )
# ----------------------------------------------------------------------------- server subprocess / external prog
class SubprocessDataProvider( base.DataProvider ):
"""
Data provider that uses the output from an intermediate program and
subprocess as it's data source.
"""
#TODO: need better ways of checking returncode, stderr for errors and raising
def __init__( self, *args, **kwargs ):
"""
:param args: the list of strings used to build commands.
:type args: variadic function args
"""
self.exit_code = None
command_list = args
self.popen = self.subprocess( *command_list, **kwargs )
#TODO:?? not communicate()?
super( SubprocessDataProvider, self ).__init__( self.popen.stdout )
self.exit_code = self.popen.poll()
#NOTE: there's little protection here v. sending a ';' and a dangerous command here
# but...we're all adults here, right? ...RIGHT?!
def subprocess( self, *command_list, **kwargs ):
"""
:param args: the list of strings used as commands.
:type args: variadic function args
"""
try:
# how expensive is this?
popen = subprocess.Popen( command_list, stderr=subprocess.PIPE, stdout=subprocess.PIPE )
log.info( 'opened subrocess (%s), PID: %s' %( str( command_list ), str( popen.pid ) ) )
#log.debug( 'stderr:\n%s\n' %( popen.stderr.read() ) )
except OSError, os_err:
command_str = ' '.join( self.command )
raise OSError( ' '.join([ str( os_err ), ':', command_str ]) )
return popen
def __exit__( self, *args ):
# poll the subrocess for an exit code
self.exit_code = self.popen.poll()
log.info( '%s.__exit__, exit_code: %s' %( str( self ), str( self.exit_code ) ) )
return super( SubprocessDataProvider, self ).__exit__( *args )
def __str__( self ):
# provide the pid and current return code
source_str = ''
if hasattr( self, 'popen' ):
source_str = '%s:%s' %( str( self.popen.pid ), str( self.popen.poll() ) )
return '%s(%s)' %( self.__class__.__name__, str( source_str ) )
class RegexSubprocessDataProvider( line.RegexLineDataProvider ):
"""
RegexLineDataProvider that uses a SubprocessDataProvider as it's data source.
"""
# this is a conv. class and not really all that necc...
def __init__( self, *args, **kwargs ):
# using subprocess as proxy data source in filtered line prov.
subproc_provider = SubprocessDataProvider( *args )
super( RegexSubprocessDataProvider, self ).__init__( subproc_provider, **kwargs )
# ----------------------------------------------------------------------------- other apis
class URLDataProvider( base.DataProvider ):
"""
Data provider that uses the contents of a URL for it's data source.
This can be piped through other providers (column, map, genome region, etc.).
"""
VALID_METHODS = ( 'GET', 'POST' )
def __init__( self, url, method='GET', data=None, **kwargs ):
"""
:param url: the base URL to open.
:param method: the HTTP method to use.
Optional: defaults to 'GET'
:param data: any data to pass (either in query for 'GET'
or as post data with 'POST')
:type data: dict
"""
self.url = url
self.method = method
self.data = data or {}
encoded_data = urllib.urlencode( self.data )
if method == 'GET':
self.url += '?%s' %( encoded_data )
opened = urllib2.urlopen( url )
elif method == 'POST':
opened = urllib2.urlopen( url, encoded_data )
else:
raise ValueError( 'Not a valid method: %s' %( method ) )
super( URLDataProvider, self ).__init__( opened, **kwargs )
#NOTE: the request object is now accessible as self.source
def __enter__( self ):
pass
def __exit__( self, *args ):
self.source.close()
# ----------------------------------------------------------------------------- generic compression
class GzipDataProvider( base.DataProvider ):
"""
Data provider that uses g(un)zip on a file as it's source.
This can be piped through other providers (column, map, genome region, etc.).
"""
def __init__( self, source, **kwargs ):
unzipped = gzip.GzipFile( source, 'rb' )
super( GzipDataProvider, self ).__init__( unzipped, **kwargs )
#NOTE: the GzipFile is now accessible in self.source
# ----------------------------------------------------------------------------- intermediate tempfile
class TempfileDataProvider( base.DataProvider ):
"""
Writes the data from the given source to a temp file, allowing
it to be used as a source where a file_name is needed (e.g. as a parameter
to a command line tool: samtools view -t <this_provider.source.file_name>)
"""
def __init__( self, source, **kwargs ):
#TODO:
raise NotImplementedError()
# write the file here
self.create_file
super( TempfileDataProvider, self ).__init__( self.tmp_file, **kwargs )
def create_file( self ):
self.tmp_file = tempfile.NamedTemporaryFile()
return self.tmp_file
def write_to_file( self ):
parent_gen = super( TempfileDataProvider, self ).__iter__()
#???
with open( self.tmp_file, 'w' ) as open_file:
for datum in parent_gen:
open_file.write( datum + '\n' )
lib/galaxy/datatypes/dataproviders/line.py
+267
View File
@@ -0,0 +1,267 @@
"""
Dataproviders that iterate over lines from their sources.
"""
import collections
import os
import re
import base
_TODO = """
line offsets (skip to some place in a file) needs to work more efficiently than simply iterating till we're there
capture tell() when provider is done
def stop( self ): self.endpoint = source.tell(); raise StopIteration()
a lot of the hierarchy here could be flattened since we're implementing pipes
"""
import logging
log = logging.getLogger( __name__ )
# ----------------------------------------------------------------------------- text
class FilteredLineDataProvider( base.LimitedOffsetDataProvider ):
"""
Data provider that yields lines of data from it's source allowing
optional control over which line to start on and how many lines
to return.
"""
DEFAULT_COMMENT_CHAR = '#'
settings = {
'string_lines' : 'bool',
'provide_blank' : 'bool',
'comment_char' : 'str',
}
def __init__( self, source, strip_lines=True, provide_blank=False, comment_char=DEFAULT_COMMENT_CHAR, **kwargs ):
"""
:param strip_lines: remove whitespace from the beginning an ending
of each line (or not).
Optional: defaults to True
:type strip_lines: bool
:param provide_blank: are empty lines considered valid and provided?
Optional: defaults to False
:type provide_blank: bool
:param comment_char: character(s) that indicate a line isn't data (a comment)
and should not be provided.
Optional: defaults to '#'
:type comment_char: str
"""
super( FilteredLineDataProvider, self ).__init__( source, **kwargs )
self.strip_lines = strip_lines
self.provide_blank = provide_blank
self.comment_char = comment_char
def filter( self, line ):
"""
Determines whether to provide line or not.
:param line: the incoming line from the source
:type line: str
:returns: a line or `None`
"""
line = super( FilteredLineDataProvider, self ).filter( line )
if line != None:
# is this the proper order?
if self.strip_lines:
line = line.strip()
if not self.provide_blank and line == '':
return None
elif line.startswith( self.comment_char ):
return None
return line
class RegexLineDataProvider( FilteredLineDataProvider ):
"""
Data provider that yields only those lines of data from it's source
that do (or do not when `invert` is True) match one or more of the given list
of regexs.
.. note:: the regex matches are effectively OR'd (if **any** regex matches
the line it is considered valid and will be provided).
"""
settings = {
'regex_list' : 'list:str',
'invert' : 'bool',
}
def __init__( self, source, regex_list=None, invert=False, **kwargs ):
"""
:param regex_list: list of strings or regular expression strings that will
be `match`ed to each line
Optional: defaults to `None` (no matching)
:type regex_list: list (of str)
:param invert: if `True` will provide only lines that **do not match**.
Optional: defaults to False
:type invert: bool
"""
super( RegexLineDataProvider, self ).__init__( source, **kwargs )
self.regex_list = regex_list if isinstance( regex_list, list ) else []
self.compiled_regex_list = [ re.compile( regex ) for regex in self.regex_list ]
self.invert = invert
#NOTE: no support for flags
def filter( self, line ):
line = super( RegexLineDataProvider, self ).filter( line )
if line != None and self.compiled_regex_list:
line = self.filter_by_regex( line )
return line
def filter_by_regex( self, line ):
matches = any([ regex.match( line ) for regex in self.compiled_regex_list ])
if self.invert:
return line if not matches else None
return line if matches else None
# ============================================================================= MICELLAINEOUS OR UNIMPLEMENTED
# ----------------------------------------------------------------------------- block data providers
class BlockDataProvider( base.LimitedOffsetDataProvider ):
"""
Class that uses formats where multiple lines combine to describe a single
datum. The data output will be a list of either map/dicts or sub-arrays.
Uses FilteredLineDataProvider as it's source (kwargs **not** passed).
e.g. Fasta, GenBank, MAF, hg log
Note: mem intensive (gathers list of lines before output)
"""
def __init__( self, source, new_block_delim_fn, block_filter_fn=None, **kwargs ):
"""
:param new_block_delim_fn: T/F function to determine whether a given line
is the start of a new block.
:type new_block_delim_fn: function
:param block_filter_fn: function that determines if a block is valid and
will be provided.
Optional: defaults to `None` (no filtering)
:type block_filter_fn: function
"""
# composition - not inheritance
#TODO: don't pass any?
line_provider = FilteredLineDataProvider( source )
super( BlockDataProvider, self ).__init__( line_provider, **kwargs )
self.new_block_delim_fn = new_block_delim_fn
self.block_filter_fn = block_filter_fn
self.init_new_block()
# ...well, this is kinda lame - but prevents returning first empty block
#TODO: maybe better way in iter
self.is_inside_block = False
def init_new_block( self ):
"""
Set up internal data for next block.
"""
# called in __init__ and after yielding the prev. block
self.block_lines = collections.deque([])
self.block = {}
def __iter__( self ):
"""
Overridden to provide last block.
"""
parent_gen = super( BlockDataProvider, self ).__iter__()
for block in parent_gen:
yield block
last_block = self.filter_block( self.assemble_current_block() )
if last_block != None and self.num_data_returned < self.limit:
self.num_data_returned += 1
yield last_block
def filter( self, line ):
"""
Line filter here being used to aggregate/assemble lines into a block
and determine whether the line indicates a new block.
:param line: the incoming line from the source
:type line: str
:returns: a block or `None`
"""
line = super( BlockDataProvider, self ).filter( line )
if line == None:
return None
if self.is_new_block( line ):
# if we're already in a block, return the prev. block and add the line to a new block
#TODO: maybe better way in iter
if self.is_inside_block:
filtered_block = self.filter_block( self.assemble_current_block() )
self.init_new_block()
self.add_line_to_block( line )
# return an assembled block datum if it passed the filter
if filtered_block != None:
return filtered_block
else:
self.is_inside_block = True
self.add_line_to_block( line )
return None
def is_new_block( self, line ):
"""
Returns True if the given line indicates the start of a new block
(and the current block should be provided) or False if not.
"""
if self.new_block_delim_fn:
return self.new_block_delim_fn( line )
return False
# NOTE:
# some formats have one block attr per line
# some formats rely on having access to multiple lines to make sensible data
# So, building the block from the lines can happen in either:
# add_line_to_block AND/OR assemble_current_block
def add_line_to_block( self, line ):
"""
Integrate the given line into the current block.
Called per line.
"""
# here either:
# consume the line (using it to add attrs to self.block)
# save the line (appending to self.block_lines) for use in assemble_current_block
self.block_lines.append( line )
def assemble_current_block( self ):
"""
Build the current data into a block.
Called per block (just before providing).
"""
# empty block_lines and assemble block
# NOTE: we don't want to have mem == 2*data here so - careful
return list( ( self.block_lines.popleft() for i in xrange( len( self.block_lines ) ) ) )
def filter_block( self, block ):
"""
Is the current block a valid/desired datum.
Called per block (just before providing).
"""
if self.block_filter_fn:
return self.block_filter_fn( block )
return block
# ----------------------------------------------------------------------------- hierarchal/tree data providers
class HierarchalDataProvider( BlockDataProvider ):
"""
Class that uses formats where a datum may have a parent or children
data.
e.g. XML, HTML, GFF3, Phylogenetic
"""
def __init__( self, source, **kwargs ):
#TODO: (and defer to better (than I can write) parsers for each subtype)
raise NotImplementedError( 'Abstract class' )
super( HierarchalDataProvider, self ).__init__( source, **kwargs )
lib/galaxy/datatypes/interval.py
+70
View File
@@ -17,6 +17,7 @@ from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.tabular import Tabular
from galaxy.datatypes.util.gff_util import parse_gff_attributes
import math
import dataproviders
log = logging.getLogger(__name__)
@@ -42,6 +43,7 @@ for key, value in alias_spec.items():
VIEWPORT_READLINE_BUFFER_SIZE = 1048576 # 1MB
VIEWPORT_MAX_READS_PER_LINE = 10
@dataproviders.decorators.has_dataproviders
class Interval( Tabular ):
"""Tab delimited data containing interval information"""
file_ext = "interval"
@@ -331,6 +333,30 @@ class Interval( Tabular ):
def get_track_resolution( self, dataset, start, end):
return None
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.GenomicRegionDataProvider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-dict',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dict_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.genomic_region_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'interval',
dataproviders.dataset.IntervalDataProvider.settings )
def interval_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.IntervalDataProvider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'interval-dict',
dataproviders.dataset.IntervalDataProvider.settings )
def interval_dict_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.interval_dataprovider( dataset, **settings )
class BedGraph( Interval ):
"""Tab delimited chrom/start/end/datavalue dataset"""
@@ -565,6 +591,8 @@ class _RemoteCallMixin:
link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url )
return link
@dataproviders.decorators.has_dataproviders
class Gff( Tabular, _RemoteCallMixin ):
"""Tab delimited data in Gff format"""
file_ext = "gff"
@@ -783,6 +811,31 @@ class Gff( Tabular, _RemoteCallMixin ):
except:
return False
# ------------- Dataproviders
# redefine bc super is Tabular
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.GenomicRegionDataProvider( dataset, 0, 3, 4, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-dict',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dict_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.genomic_region_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'interval',
dataproviders.dataset.IntervalDataProvider.settings )
def interval_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.IntervalDataProvider( dataset, 0, 3, 4, 6, 2, **settings )
@dataproviders.decorators.dataprovider_factory( 'interval-dict',
dataproviders.dataset.IntervalDataProvider.settings )
def interval_dict_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.interval_dataprovider( dataset, **settings )
class Gff3( Gff ):
"""Tab delimited data in Gff3 format"""
file_ext = "gff3"
@@ -960,6 +1013,7 @@ class Gtf( Gff ):
except:
return False
@dataproviders.decorators.has_dataproviders
class Wiggle( Tabular, _RemoteCallMixin ):
"""Tab delimited data in wiggle format"""
file_ext = "wig"
@@ -1146,6 +1200,19 @@ class Wiggle( Tabular, _RemoteCallMixin ):
resolution = max( resolution, 1 )
return resolution
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'wiggle', dataproviders.dataset.WiggleDataProvider.settings )
def wiggle_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.dataset.WiggleDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'wiggle-dict', dataproviders.dataset.WiggleDataProvider.settings )
def wiggle_dict_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
settings[ 'named_columns' ] = True
return dataproviders.dataset.WiggleDataProvider( dataset_source, **settings )
class CustomTrack ( Tabular ):
"""UCSC CustomTrack"""
file_ext = "customtrack"
@@ -1272,6 +1339,7 @@ class CustomTrack ( Tabular ):
return False
return True
class ENCODEPeak( Interval ):
'''
Human ENCODE peak format. There are both broad and narrow peak formats.
@@ -1300,6 +1368,7 @@ class ENCODEPeak( Interval ):
def sniff( self, filename ):
return False
class ChromatinInteractions( Interval ):
'''
Chromatin interactions obtained from 3C/5C/Hi-C experiments.
@@ -1325,6 +1394,7 @@ class ChromatinInteractions( Interval ):
def sniff( self, filename ):
return False
if __name__ == '__main__':
import doctest, sys
doctest.testmod(sys.modules[__name__])
lib/galaxy/datatypes/registry.py
+4 -2
View File
@@ -121,6 +121,8 @@ class Registry( object ):
# TODO: Handle deactivating datatype converters, etc before removing from self.datatypes_by_extension.
self.log.debug( "Removing datatype with extension '%s' from the registry." % extension )
del self.datatypes_by_extension[ extension ]
if extension in self.upload_file_formats:
self.upload_file_formats.remove( extension )
can_process_datatype = False
else:
can_process_datatype = ( extension and ( dtype or type_extension ) ) and ( extension not in self.datatypes_by_extension or override )
@@ -165,7 +167,7 @@ class Registry( object ):
self.mimetypes_by_extension[ extension ] = mimetype
if datatype_class.track_type:
self.available_tracks.append( extension )
if display_in_upload:
if display_in_upload and extension not in self.upload_file_formats:
self.upload_file_formats.append( extension )
# Max file size cut off for setting optional metadata
self.datatypes_by_extension[ extension ].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None )
@@ -613,7 +615,7 @@ class Registry( object ):
def find_conversion_destination_for_dataset_by_extensions( self, dataset, accepted_formats, converter_safe = True ):
"""Returns ( target_ext, existing converted dataset )"""
for convert_ext in self.get_converters_by_datatype( dataset.ext ):
if isinstance( self.get_datatype_by_extension( convert_ext ), accepted_formats ):
if self.get_datatype_by_extension( convert_ext ).matches_any( accepted_formats ):
converted_dataset = dataset.get_converted_files_by_type( convert_ext )
if converted_dataset:
ret_data = converted_dataset
lib/galaxy/datatypes/sequence.py
+6
View File
@@ -397,6 +397,7 @@ class Fasta( Sequence ):
f.close()
_count_split = classmethod(_count_split)
class csFasta( Sequence ):
""" Class representing the SOLID Color-Space sequence ( csfasta ) """
file_ext = "csfasta"
@@ -445,6 +446,7 @@ class csFasta( Sequence ):
return
return Sequence.set_meta( self, dataset, **kwd )
class Fastq ( Sequence ):
"""Class representing a generic FASTQ sequence"""
file_ext = "fastq"
@@ -660,6 +662,7 @@ def COPIED_build_maf_index_species_chromosomes( filename, index_species = None )
return ( None, [], {}, 0 )
return ( indexes, species, species_chromosomes, blocks )
class Maf( Alignment ):
"""Class describing a Maf alignment"""
file_ext = "maf"
@@ -766,6 +769,7 @@ class Maf( Alignment ):
except:
return False
class MafCustomTrack( data.Text ):
file_ext = "mafcustomtrack"
@@ -803,6 +807,7 @@ class MafCustomTrack( data.Text ):
except:
pass
class Axt( data.Text ):
"""Class describing an axt alignment"""
@@ -855,6 +860,7 @@ class Axt( data.Text ):
else:
return True
class Lav( data.Text ):
"""Class describing a LAV alignment"""
lib/galaxy/datatypes/sniff.py
+1
View File
@@ -6,6 +6,7 @@ import registry
from galaxy import util
from galaxy.datatypes.checkers import *
from encodings import search_function as encodings_search_function
from binary import Binary
log = logging.getLogger(__name__)
lib/galaxy/datatypes/tabular.py
+126 -4
View File
@@ -15,9 +15,11 @@ from galaxy.datatypes.checkers import is_gzip
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.sniff import get_headers, get_test_fname
from galaxy.util.json import to_json_string
import dataproviders
log = logging.getLogger(__name__)
@dataproviders.decorators.has_dataproviders
class Tabular( data.Text ):
"""Tab delimited data"""
@@ -342,6 +344,31 @@ class Tabular( data.Text ):
return vizs
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings )
def column_dataprovider( self, dataset, **settings ):
"""Uses column settings that are passed in"""
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.column.ColumnarDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'dataset-column',
dataproviders.column.ColumnarDataProvider.settings )
def dataset_column_dataprovider( self, dataset, **settings ):
"""Attempts to get column settings from dataset.metadata"""
return dataproviders.dataset.DatasetColumnarDataProvider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'dict', dataproviders.column.DictDataProvider.settings )
def dict_dataprovider( self, dataset, **settings ):
"""Uses column settings that are passed in"""
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.column.DictDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'dataset-dict', dataproviders.column.DictDataProvider.settings )
def dataset_dict_dataprovider( self, dataset, **settings ):
"""Attempts to get column settings from dataset.metadata"""
return dataproviders.dataset.DatasetDictDataProvider( dataset, **settings )
class Taxonomy( Tabular ):
def __init__(self, **kwd):
"""Initialize taxonomy datatype"""
@@ -355,6 +382,8 @@ class Taxonomy( Tabular ):
"""Returns formated html of peek"""
return Tabular.make_html_table( self, dataset, column_names=self.column_names )
@dataproviders.decorators.has_dataproviders
class Sam( Tabular ):
file_ext = 'sam'
track_type = "ReadTrack"
@@ -469,6 +498,72 @@ class Sam( Tabular ):
raise Exception('Result %s from %s' % (result, cmd))
merge = staticmethod(merge)
# ------------- Dataproviders
# sam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers
#TODO:?? seems like there should be an easier way to do this - metadata.comment_char?
@dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings )
def line_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).line_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'regex-line', dataproviders.line.RegexLineDataProvider.settings )
def regex_line_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).regex_line_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings )
def column_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).column_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'dataset-column',
dataproviders.column.ColumnarDataProvider.settings )
def dataset_column_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).dataset_column_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'dict', dataproviders.column.DictDataProvider.settings )
def dict_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).dict_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'dataset-dict', dataproviders.column.DictDataProvider.settings )
def dataset_dict_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).dataset_dict_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'header', dataproviders.line.RegexLineDataProvider.settings )
def header_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
headers_source = dataproviders.line.RegexLineDataProvider( dataset_source, regex_list=[ '^@' ] )
return dataproviders.line.RegexLineDataProvider( headers_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'id-seq-qual', dict_dataprovider.settings )
def id_seq_qual_dataprovider( self, dataset, **settings ):
# provided as an example of a specified column dict (w/o metadata)
settings[ 'indeces' ] = [ 0, 9, 10 ]
settings[ 'column_names' ] = [ 'id', 'seq', 'qual' ]
return self.dict_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return dataproviders.dataset.GenomicRegionDataProvider( dataset, 2, 3, 3, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-dict',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dict_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return dataproviders.dataset.GenomicRegionDataProvider( dataset, 2, 3, 3, True, **settings )
#@dataproviders.decorators.dataprovider_factory( 'samtools' )
#def samtools_dataprovider( self, dataset, **settings ):
# dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
# return dataproviders.dataset.SamtoolsDataProvider( dataset_source, **settings )
@dataproviders.decorators.has_dataproviders
class Pileup( Tabular ):
"""Tab delimited data in pileup (6- or 10-column) format"""
file_ext = "pileup"
@@ -527,12 +622,20 @@ class Pileup( Tabular ):
except:
return False
class ElandMulti( Tabular ):
file_ext = 'elandmulti'
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.GenomicRegionDataProvider( dataset, **settings )
def sniff( self, filename ):
return False
@dataproviders.decorators.dataprovider_factory( 'genomic-region-dict',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dict_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.genomic_region_dataprovider( dataset, **settings )
@dataproviders.decorators.has_dataproviders
class Vcf( Tabular ):
""" Variant Call Format for describing SNPs and other simple genome variations. """
track_type = "VariantTrack"
@@ -568,6 +671,18 @@ class Vcf( Tabular ):
# Found header line, get sample names.
dataset.metadata.sample_names = line.split()[ 9: ]
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.GenomicRegionDataProvider( dataset, 0, 1, 1, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-dict',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dict_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.genomic_region_dataprovider( dataset, **settings )
class Eland( Tabular ):
"""Support for the export.txt.gz file used by Illumina's ELANDv2e aligner"""
@@ -699,6 +814,13 @@ class Eland( Tabular ):
dataset.metadata.reads = reads.keys()
class ElandMulti( Tabular ):
file_ext = 'elandmulti'
def sniff( self, filename ):
return False
class FeatureLocationIndex( Tabular ):
"""
An index that stores feature locations in tabular format.
lib/galaxy/jobs/__init__.py
+4 -3
View File
@@ -692,7 +692,10 @@ class JobWrapper( object ):
incoming['__user_email__'] = incoming['userEmail'] = user_email
incoming['__user_name__'] = user_name
# Build params, done before hook so hook can use
param_dict = self.tool.build_param_dict( incoming, inp_data, out_data, self.get_output_fnames(), self.working_directory )
param_dict = self.tool.build_param_dict( incoming,
inp_data, out_data,
self.get_output_fnames(),
self.working_directory )
# Certain tools require tasks to be completed prior to job execution
# ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ).
self.tool.exec_before_job( self.queue.app, inp_data, out_data, param_dict )
@@ -919,8 +922,6 @@ class JobWrapper( object ):
return self.fail( "Job %s's output dataset(s) could not be read" % job.id )
job_context = ExpressionContext( dict( stdout = job.stdout, stderr = job.stderr ) )
#DBTODO unused
#job_tool = self.app.toolbox.tools_by_id.get( job.tool_id, None )
for dataset_assoc in job.output_datasets + job.output_library_datasets:
context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset )
#should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur
lib/galaxy/jobs/handler.py
+1 -1
View File
@@ -10,7 +10,7 @@ from Queue import Queue, Empty
from sqlalchemy.sql.expression import and_, or_, select, func
from galaxy import util, model
from galaxy import model
from galaxy.jobs import Sleeper, JobWrapper, TaskWrapper, JobDestination
log = logging.getLogger( __name__ )
lib/galaxy/jobs/runners/condor.py
+17 -6
View File
@@ -32,6 +32,12 @@ cd %s
%s
"""
default_query_classad = dict(
universe = 'vanilla',
getenv = 'true',
notification = 'NEVER',
)
class CondorJobState( AsynchronousJobState ):
def __init__( self, **kwargs ):
"""
@@ -71,6 +77,10 @@ class CondorJobRunner( AsynchronousJobRunner ):
# wrapper.get_id_tag() instead of job_id for compatibility with TaskWrappers.
galaxy_id_tag = job_wrapper.get_id_tag()
# get destination params
query_params = default_query_classad.copy()
query_params.update( job_destination.params )
# define job attributes
cjs = CondorJobState( files_dir=self.app.config.cluster_files_directory, job_wrapper=job_wrapper )
cjs.user_log = os.path.join( self.app.config.cluster_files_directory, 'galaxy_%s.condor.log' % galaxy_id_tag )
@@ -78,13 +88,12 @@ class CondorJobRunner( AsynchronousJobRunner ):
submit_file = os.path.join( self.app.config.cluster_files_directory, 'galaxy_%s.condor.desc' % galaxy_id_tag )
executable = cjs.job_file
submit_desc = [ ]
submit_desc.append( 'universe = vanilla' )
submit_desc.append( 'getenv = true' )
for k, v in query_params.items():
submit_desc.append( '%s = %s' % ( k, v ) )
submit_desc.append( 'executable = ' + executable )
submit_desc.append( 'output = ' + cjs.output_file )
submit_desc.append( 'error = ' + cjs.error_file )
submit_desc.append( 'log = ' + cjs.user_log )
submit_desc.append( 'notification = NEVER' )
submit_desc.append( 'queue' )
script = drm_template % (job_wrapper.galaxy_lib_dir, os.path.abspath( job_wrapper.working_directory ), command_line)
@@ -138,15 +147,16 @@ class CondorJobRunner( AsynchronousJobRunner ):
# TODO Add extra except for OSError?
s_out = str(e)
os.unlink( submit_file )
if external_job_id is None:
log.debug( "condor_submit failed for job %s: %s" % (job_wrapper.get_id_tag(), s_out) )
if self.app.config.cleanup_job == "always":
os.unlink( submit_file )
cjs.cleanup()
job_wrapper.fail( "condor_submit failed", exception=True )
return
os.unlink( submit_file )
log.info( "(%s) queued as %s" % ( galaxy_id_tag, external_job_id ) )
# store runner information for tracking if Galaxy restarts
@@ -232,6 +242,7 @@ class CondorJobRunner( AsynchronousJobRunner ):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
# TODO Check if we need any changes here
job_id = job.get_job_runner_external_id()
galaxy_id_tag = job_wrapper.get_id_tag()
if job_id is None:
self.put( job_wrapper )
return
@@ -252,7 +263,7 @@ class CondorJobRunner( AsynchronousJobRunner ):
cjs.running = False
self.monitor_queue.put( cjs )
def __old_state_paths( self, cjs, job ):
def __old_state_paths( self, cjs ):
"""For recovery of jobs started prior to standardizing the naming of
files in the AsychronousJobState object
"""
lib/galaxy/jobs/runners/lwr.py
+28 -18
View File
@@ -14,6 +14,7 @@ log = logging.getLogger( __name__ )
__all__ = [ 'LwrJobRunner' ]
class LwrJobRunner( AsynchronousJobRunner ):
"""
LWR Job Runner
@@ -49,18 +50,17 @@ class LwrJobRunner( AsynchronousJobRunner ):
return job_state
def queue_job(self, job_wrapper):
stderr = stdout = command_line = ''
command_line = ''
job_destination = job_wrapper.job_destination
try:
job_wrapper.prepare()
if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None:
for cmd in job_wrapper.prepare_input_files_cmds: # run the commands to stage the input files
for cmd in job_wrapper.prepare_input_files_cmds: # run the commands to stage the input files
#log.debug( 'executing: %s' % cmd )
if 0 != os.system(cmd):
raise Exception('Error running file staging command: %s' % cmd)
job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line
job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line
command_line = self.build_command_line( job_wrapper, include_metadata=False, include_work_dir_outputs=False )
except:
job_wrapper.fail( "failure preparing job", exception=True )
@@ -73,7 +73,6 @@ class LwrJobRunner( AsynchronousJobRunner ):
return
try:
#log.debug( 'executing: %s' % command_line )
client = self.get_client_from_wrapper(job_wrapper)
output_files = self.get_output_files(job_wrapper)
input_files = job_wrapper.get_input_fnames()
@@ -85,8 +84,7 @@ class LwrJobRunner( AsynchronousJobRunner ):
client.launch( rebuilt_command_line )
job_wrapper.set_job_destination( job_destination, job_id )
job_wrapper.change_state( model.Job.states.QUEUED )
except Exception, exc:
except:
job_wrapper.fail( "failure running job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
@@ -124,23 +122,36 @@ class LwrJobRunner( AsynchronousJobRunner ):
client = self.get_client_from_state(job_state)
run_results = client.raw_check_complete()
log.debug('run_results %s' % run_results )
stdout = run_results['stdout']
stderr = run_results['stderr']
download_failure_exceptions = []
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ]:
work_dir_outputs = self.get_work_dir_outputs(job_wrapper)
output_files = self.get_output_files(job_wrapper)
for source_file, output_file in work_dir_outputs:
client.download_work_dir_output(source_file, job_wrapper.working_directory, output_file)
try:
client.download_work_dir_output(source_file, job_wrapper.working_directory, output_file)
except Exception, e:
download_failure_exceptions.append(e)
# Remove from full output_files list so don't try to download directly.
output_files.remove(output_file)
for output_file in output_files:
client.download_output(output_file, working_directory=job_wrapper.working_directory)
client.clean()
try:
client.download_output(output_file, working_directory=job_wrapper.working_directory)
except Exception, e:
download_failure_exceptions.append(e)
if download_failure_exceptions or self.app.config.cleanup_job == "always":
try:
client.clean()
except:
log.warn("Failed to cleanup remote LWR job")
if download_failure_exceptions:
job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True)
log.debug('execution finished: %s' % command_line)
except Exception, exc:
job_wrapper.fail( "failure running job", exception=True )
except:
message = "Failed to communicate with remote job server."
job_wrapper.fail( message, exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
self._handle_metadata_externally( job_wrapper )
@@ -166,14 +177,14 @@ class LwrJobRunner( AsynchronousJobRunner ):
if e.errno == errno.ESRCH:
log.debug( "check_pid(): PID %d is dead" % pid )
else:
log.warning( "check_pid(): Got errno %s when attempting to check PID %d: %s" %( errno.errorcode[e.errno], pid, e.strerror ) )
log.warning( "check_pid(): Got errno %s when attempting to check PID %d: %s" % ( errno.errorcode[e.errno], pid, e.strerror ) )
return False
def stop_job( self, job ):
#if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished
job_ext_output_metadata = job.get_external_output_metadata()
if job_ext_output_metadata:
pid = job_ext_output_metadata[0].job_runner_external_pid #every JobExternalOutputMetadata has a pid set, we just need to take from one of them
pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them
if pid in [ None, '' ]:
log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.id )
return
@@ -189,10 +200,10 @@ class LwrJobRunner( AsynchronousJobRunner ):
return # give up
sleep( 2 )
if not self.check_pid( pid ):
log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" %( job.id, pid, sig ) )
log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" % ( job.id, pid, sig ) )
return
else:
log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" %( job.id, pid ) )
log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job.id, pid ) )
else:
# Remote kill
lwr_url = job.job_runner_name
@@ -201,7 +212,6 @@ class LwrJobRunner( AsynchronousJobRunner ):
client = self.get_client(job.destination_params, job_id)
client.kill()
def recover( self, job, job_wrapper ):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
job_state = AsynchronousJobState()
lib/galaxy/jobs/runners/lwr_client/client.py
+10 -1
View File
@@ -19,6 +19,15 @@ class parseJson(object):
return replacement
class OutputNotFoundException(Exception):
def __init__(self, path):
self.path = path
def __str__(self):
return "No remote output found for path %s" % self.path
class Client(object):
"""
Objects of this client class perform low-level communication with a remote LWR server.
@@ -163,7 +172,7 @@ class Client(object):
elif output_type == "task":
output_path = os.path.join(working_directory, name)
else:
raise Exception("No remote output found for dataset with path %s" % path)
raise OutputNotFoundException(path)
self.__raw_download_output(name, self.job_id, output_type, output_path)
def __raw_download_output(self, name, job_id, output_type, output_path):
lib/galaxy/model/__init__.py
+35 -3
View File
@@ -157,7 +157,10 @@ class User( object, APIItem ):
return total
class Job( object ):
class Job( object, APIItem ):
api_collection_visible_keys = [ 'id' ]
api_element_visible_keys = [ 'id' ]
"""
A job represents a request to run a tool given input datasets, tool
parameters, and output datasets.
@@ -360,6 +363,34 @@ class Job( object ):
dataset.blurb = 'deleted'
dataset.peek = 'Job deleted'
dataset.info = 'Job output deleted by user before job completed'
def get_api_value( self, view='collection' ):
rval = super( Job, self ).get_api_value( view=view )
rval['tool_name'] = self.tool_id
param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] )
rval['params'] = param_dict
input_dict = {}
for i in self.input_datasets:
if i.dataset is not None:
input_dict[i.name] = {"hda_id" : i.dataset.id}
for i in self.input_library_datasets:
if i.dataset is not None:
input_dict[i.name] = {"ldda_id" : i.dataset.id}
for k in input_dict:
if k in param_dict:
del param_dict[k]
rval['inputs'] = input_dict
output_dict = {}
for i in self.output_datasets:
if i.dataset is not None:
output_dict[i.name] = {"hda_id" : i.dataset.id}
for i in self.output_library_datasets:
if i.dataset is not None:
output_dict[i.name] = {"ldda_id" : i.dataset.id}
rval['outputs'] = output_dict
return rval
class Task( object ):
"""
@@ -1542,7 +1573,7 @@ class HistoryDatasetAssociation( DatasetInstance, UsesAnnotations ):
return hda
def to_library_dataset_dataset_association( self, trans, target_folder,
replace_dataset=None, parent_id=None, user=None, roles=[], ldda_message='' ):
replace_dataset=None, parent_id=None, user=None, roles=None, ldda_message='' ):
"""
Copy this HDA to a library optionally replacing an existing LDDA.
"""
@@ -1577,6 +1608,7 @@ class HistoryDatasetAssociation( DatasetInstance, UsesAnnotations ):
object_session( self ).add( ldda )
object_session( self ).flush()
# If roles were selected on the upload form, restrict access to the Dataset to those roles
roles = roles or []
for role in roles:
dp = trans.model.DatasetPermissions( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action,
ldda.dataset, role )
@@ -3537,7 +3569,7 @@ class ToolShedRepository( object ):
def repository_dependencies_being_installed( self ):
required_repositories_being_installed = []
for required_repository in self.repository_dependencies:
if tool_dependency.status == ToolDependency.installation_status.INSTALLING:
if required_repository.status == self.installation_status.INSTALLING:
required_repositories_being_installed.append( required_repository )
return required_repositories_being_installed
@property
lib/galaxy/model/mapping.py
+19 -13
View File
@@ -16,6 +16,7 @@ from sqlalchemy.orm import backref, object_session, relation, scoped_session, se
from sqlalchemy.orm.collections import attribute_mapped_collection
from galaxy import model
from galaxy.model.orm import dialect_to_egg
from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString, UUIDType
from galaxy.security import GalaxyRBACAgent
from galaxy.util.bunch import Bunch
@@ -29,11 +30,6 @@ context = Session = scoped_session( sessionmaker( autoflush=False, autocommit=Tr
# For backward compatibility with "context.current"
context.current = Session
dialect_to_egg = {
"sqlite" : "pysqlite>=2",
"postgres" : "psycopg2",
"mysql" : "MySQL_python"
}
# NOTE REGARDING TIMESTAMPS:
# It is currently difficult to have the timestamps calculated by the
@@ -1616,17 +1612,27 @@ mapper( model.LibraryDatasetDatasetInfoAssociation, model.LibraryDatasetDatasetI
primaryjoin=( model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.table.c.id ) )
) )
mapper( model.JobToInputDatasetAssociation, model.JobToInputDatasetAssociation.table,
properties=dict( job=relation( model.Job ), dataset=relation( model.HistoryDatasetAssociation, lazy=False, backref="dependent_jobs" ) ) )
mapper( model.JobToInputDatasetAssociation,
model.JobToInputDatasetAssociation.table, properties=dict(
job=relation( model.Job ), dataset=relation(
model.HistoryDatasetAssociation, lazy=False,
backref="dependent_jobs" ) ) )
mapper( model.JobToOutputDatasetAssociation, model.JobToOutputDatasetAssociation.table,
properties=dict( job=relation( model.Job ), dataset=relation( model.HistoryDatasetAssociation, lazy=False ) ) )
mapper( model.JobToOutputDatasetAssociation,
model.JobToOutputDatasetAssociation.table, properties=dict(
job=relation( model.Job ), dataset=relation(
model.HistoryDatasetAssociation, lazy=False ) ) )
mapper( model.JobToInputLibraryDatasetAssociation, model.JobToInputLibraryDatasetAssociation.table,
properties=dict( job=relation( model.Job ), dataset=relation( model.LibraryDatasetDatasetAssociation, lazy=False, backref="dependent_jobs" ) ) )
mapper( model.JobToInputLibraryDatasetAssociation,
model.JobToInputLibraryDatasetAssociation.table, properties=dict(
job=relation( model.Job ), dataset=relation(
model.LibraryDatasetDatasetAssociation, lazy=False,
backref="dependent_jobs" ) ) )
mapper( model.JobToOutputLibraryDatasetAssociation, model.JobToOutputLibraryDatasetAssociation.table,
properties=dict( job=relation( model.Job ), dataset=relation( model.LibraryDatasetDatasetAssociation, lazy=False ) ) )
mapper( model.JobToOutputLibraryDatasetAssociation,
model.JobToOutputLibraryDatasetAssociation.table, properties=dict(
job=relation( model.Job ), dataset=relation(
model.LibraryDatasetDatasetAssociation, lazy=False ) ) )
mapper( model.JobParameter, model.JobParameter.table )
lib/galaxy/model/migrate/check.py
+2 -5
View File
@@ -12,16 +12,13 @@ from sqlalchemy import *
from sqlalchemy.exc import NoSuchTableError
from migrate.versioning import repository, schema
from galaxy.model.orm import dialect_to_egg
log = logging.getLogger( __name__ )
# path relative to galaxy
migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 )
migrate_repository = repository.Repository( migrate_repository_directory )
dialect_to_egg = {
"sqlite" : "pysqlite>=2",
"postgres" : "psycopg2",
"mysql" : "MySQL_python"
}
def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=None ):
"""
lib/galaxy/model/migrate/versions/0014_pages.py
+13 -1
View File
@@ -41,8 +41,20 @@ def upgrade(migrate_engine):
print __doc__
metadata.reflect()
try:
if migrate_engine.name == 'mysql':
# Strip slug index prior to creation so we can do it manually.
slug_index = None
for ix in Page_table.indexes:
if ix.name == 'ix_page_slug':
slug_index = ix
Page_table.indexes.remove(slug_index)
Page_table.create()
except:
if migrate_engine.name == 'mysql':
# Create slug index manually afterward.
i = Index( "ix_page_slug", Page_table.c.slug, mysql_length = 200)
i.create()
except Exception, ex:
log.debug(ex)
log.debug( "Could not create page table" )
try:
PageRevision_table.create()
lib/galaxy/model/migrate/versions/0017_library_item_indexes.py
+1 -1
View File
@@ -32,7 +32,7 @@ def upgrade(migrate_engine):
# Load existing tables
metadata.reflect()
# Add 1 index to the library_folder table
i = Index( 'ix_library_folder_name', LibraryFolder_table.c.name )
i = Index( 'ix_library_folder_name', LibraryFolder_table.c.name, mysql_length=200 )
try:
i.create()
except Exception, e:
lib/galaxy/model/migrate/versions/0030_history_slug_column.py
+12 -3
View File
@@ -19,12 +19,21 @@ def upgrade(migrate_engine):
History_table = Table( "history", metadata, autoload=True )
# Create slug column.
c = Column( "slug", TEXT, index=True )
c.create( History_table , index_name='ix_history_slug')
# Mysql needs manual index creation because of max length index.
if migrate_engine.name != 'mysql':
# Create slug column.
c = Column( "slug", TEXT, index=True )
c.create( History_table , index_name='ix_history_slug')
else:
c = Column( "slug", TEXT )
c.create( History_table , index_name='')
i = Index( "ix_history_slug", History_table.c.slug, mysql_length = 200)
i.create()
assert c is History_table.c.slug
## Create slug index.
#try:
#i = Index( "ix_history_slug", History_table.c.slug )
lib/galaxy/model/migrate/versions/0032_stored_workflow_slug_column.py
+5 -9
View File
@@ -19,20 +19,16 @@ def upgrade(migrate_engine):
StoredWorkflow_table = Table( "stored_workflow", metadata, autoload=True )
if migrate_engine.name != 'sqlite':
# Create slug column.
c = Column( "slug", TEXT, index=True )
c.create( StoredWorkflow_table, index_name='ix_stored_workflow_slug' )
else:
# Create slug column.
c = Column( "slug", TEXT )
c.create( StoredWorkflow_table )
# Create slug column.
c = Column( "slug", TEXT )
c.create( StoredWorkflow_table )
assert c is StoredWorkflow_table.c.slug
# Create slug index.
if migrate_engine.name != 'sqlite':
try:
i = Index( "ix_stored_workflow_slug", StoredWorkflow_table.c.slug )
i = Index( "ix_stored_workflow_slug", StoredWorkflow_table.c.slug, mysql_length = 200 )
i.create()
except:
# Mysql doesn't have a named index, but alter should work
lib/galaxy/model/migrate/versions/0035_item_annotations_and_workflow_step_tags.py
+19 -4
View File
@@ -18,25 +18,25 @@ HistoryAnnotationAssociation_table = Table( "history_annotation_association", me
Column( "id", Integer, primary_key=True ),
Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "annotation", TEXT, index=True) )
Column( "annotation", TEXT ) )
HistoryDatasetAssociationAnnotationAssociation_table = Table( "history_dataset_association_annotation_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "annotation", TEXT, index=True) )
Column( "annotation", TEXT ) )
StoredWorkflowAnnotationAssociation_table = Table( "stored_workflow_annotation_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "annotation", TEXT, index=True) )
Column( "annotation", TEXT ) )
WorkflowStepAnnotationAssociation_table = Table( "workflow_step_annotation_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "annotation", TEXT, index=True) )
Column( "annotation", TEXT ) )
# Tagging tables.
@@ -49,6 +49,7 @@ WorkflowStepTagAssociation_table = Table( "workflow_step_tag_association", metad
Column( "value", Unicode(255), index=True),
Column( "user_value", Unicode(255), index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -89,6 +90,20 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating workflow_step_tag_association table failed: %s" % str( e ) )
haaa = Index( "ix_history_anno_assoc_annotation", HistoryAnnotationAssociation_table.c.annotation, mysql_length = 200)
hdaaa = Index( "ix_history_dataset_anno_assoc_annotation", HistoryDatasetAssociationAnnotationAssociation_table.c.annotation, mysql_length = 200)
swaaa = Index( "ix_stored_workflow_ann_assoc_annotation", StoredWorkflowAnnotationAssociation_table.c.annotation, mysql_length = 200)
wsaaa = Index( "ix_workflow_step_ann_assoc_annotation", WorkflowStepAnnotationAssociation_table.c.annotation, mysql_length = 200)
try:
haaa.create()
hdaaa.create()
swaaa.create()
wsaaa.create()
except Exception, e:
print str(e)
log.debug( "Creating annotation indices failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py
+9 -4
View File
@@ -23,10 +23,15 @@ def upgrade(migrate_engine):
Visualization_revision_table = Table( "visualization_revision", metadata, autoload=True )
# Create dbkey columns.
x = Column( "dbkey", TEXT, index=True )
y = Column( "dbkey", TEXT, index=True )
x.create( Visualization_table, index_name = "ix_visualization_dbkey" )
y.create( Visualization_revision_table, index_name = "ix_visualization_revision_dbkey" )
x = Column( "dbkey", TEXT )
y = Column( "dbkey", TEXT )
x.create( Visualization_table )
y.create( Visualization_revision_table )
# Manually create indexes for compatability w/ mysql_length.
xi = Index( "ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length = 200)
xi.create()
yi = Index( "ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length = 200)
yi.create()
assert x is Visualization_table.c.dbkey
assert y is Visualization_revision_table.c.dbkey
lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py
+1 -1
View File
@@ -54,7 +54,7 @@ def upgrade(migrate_engine):
log.debug( "Creating data_manager_job_association table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
try:
lib/galaxy/model/orm/__init__.py
+7
View File
@@ -7,3 +7,10 @@ from sqlalchemy.interfaces import *
import sqlalchemy.exc
from sqlalchemy.ext.orderinglist import ordering_list
dialect_to_egg = {
"sqlite" : "pysqlite>=2",
"postgres" : "psycopg2",
"postgresql" : "psycopg2",
"mysql" : "MySQL_python"
}
lib/galaxy/model/search.py
+90 -8
View File
@@ -31,10 +31,13 @@ from galaxy import eggs
eggs.require("Parsley")
import parsley
from galaxy.model import HistoryDatasetAssociation, LibraryDatasetDatasetAssociation, History, Library, LibraryFolder, LibraryDataset
from galaxy.model import StoredWorkflowTagAssociation, StoredWorkflow, HistoryTagAssociation, ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation
from galaxy.model import ToolVersion
from galaxy.model import (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation,
History, Library, LibraryFolder, LibraryDataset,StoredWorkflowTagAssociation,
StoredWorkflow, HistoryTagAssociation,HistoryDatasetAssociationTagAssociation,
ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation, Job, JobParameter,
JobToInputDatasetAssociation, JobToOutputDatasetAssociation, ToolVersion)
from galaxy.util.json import to_json_string
from sqlalchemy import and_
from sqlalchemy.orm import aliased
@@ -269,12 +272,30 @@ class ToolView(ViewQueryBaseClass):
#History Dataset Searching
##################
def history_dataset_handle_tag(view, left, operator, right):
if operator == "=":
view.do_query = True
#aliasing the tag association table, so multiple links to different tags can be formed during a single query
tag_table = aliased(HistoryDatasetAssociationTagAssociation)
view.query = view.query.filter(
HistoryDatasetAssociation.id == tag_table.history_dataset_association_id
)
tmp = right.split(":")
view.query = view.query.filter( tag_table.user_tname == tmp[0] )
if len(tmp) > 1:
view.query = view.query.filter( tag_table.user_value == tmp[1] )
else:
raise GalaxyParseError("Invalid comparison operator: %s" % (operator))
class HistoryDatasetView(ViewQueryBaseClass):
DOMAIN = "history_dataset"
FIELDS = {
'name' : ViewField('name', sqlalchemy_field=HistoryDatasetAssociation.name),
'id' : ViewField('id',sqlalchemy_field=HistoryDatasetAssociation.id, id_decode=True)
'id' : ViewField('id',sqlalchemy_field=HistoryDatasetAssociation.id, id_decode=True),
'tag' : ViewField("tag", handler=history_dataset_handle_tag)
}
def search(self, trans):
@@ -289,13 +310,14 @@ class HistoryDatasetView(ViewQueryBaseClass):
def history_handle_tag(view, left, operator, right):
if operator == "=":
view.do_query = True
tag_table = aliased(HistoryTagAssociation)
view.query = view.query.filter(
History.id == HistoryTagAssociation.history_id
History.id == tag_table.history_id
)
tmp = right.split(":")
view.query = view.query.filter( HistoryTagAssociation.user_tname == tmp[0] )
view.query = view.query.filter( tag_table.user_tname == tmp[0] )
if len(tmp) > 1:
view.query = view.query.filter( HistoryTagAssociation.user_value == tmp[1] )
view.query = view.query.filter( tag_table.user_value == tmp[1] )
else:
raise GalaxyParseError("Invalid comparison operator: %s" % (operator))
@@ -362,6 +384,65 @@ class WorkflowView(ViewQueryBaseClass):
def search(self, trans):
self.query = trans.sa_session.query( StoredWorkflow )
##################
#Job Searching
##################
def job_param_filter(view, left, operator, right):
view.do_query = True
alias = aliased( JobParameter )
param_name = re.sub(r'^param.', '', left)
view.query = view.query.filter(
and_(
Job.id == alias.job_id,
alias.name == param_name,
alias.value == to_json_string(right)
)
)
def job_input_hda_filter(view, left, operator, right):
view.do_query = True
alias = aliased( JobToInputDatasetAssociation )
param_name = re.sub(r'^input_hda.', '', left)
view.query = view.query.filter(
and_(
Job.id == alias.job_id,
alias.name == param_name,
alias.dataset_id == right
)
)
def job_output_hda_filter(view, left, operator, right):
view.do_query = True
alias = aliased( JobToOutputDatasetAssociation )
param_name = re.sub(r'^output_hda.', '', left)
view.query = view.query.filter(
and_(
Job.id == alias.job_id,
alias.name == param_name,
alias.dataset_id == right
)
)
class JobView(ViewQueryBaseClass):
DOMAIN = "job"
FIELDS = {
'tool_name' : ViewField('tool_name', sqlalchemy_field=Job.tool_id),
'param' : ViewField('param', handler=job_param_filter),
'input_hda' : ViewField('input_hda', handler=job_input_hda_filter, id_decode=True),
'output_hda' : ViewField('output_hda', handler=job_output_hda_filter, id_decode=True)
}
def search(self, trans):
self.query = trans.sa_session.query( Job )
"""
The view mapping takes a user's name for a table and maps it to a View class that will
handle queries
@@ -377,7 +458,8 @@ view_mapping = {
'hda' : HistoryDatasetView,
'history' : HistoryView,
'workflow' : WorkflowView,
'tool' : ToolView
'tool' : ToolView,
'job' : JobView,
}
"""
lib/galaxy/objectstore/__init__.py
+4 -2
View File
@@ -880,7 +880,8 @@ class DistributedObjectStore(ObjectStore):
self.__parse_distributed_config(config)
if self.global_max_percent_full or filter(lambda x: x is not None, self.max_percent_full.values()):
self.sleeper = None
if self.global_max_percent_full or filter(lambda x: x != 0.0, self.max_percent_full.values()):
self.sleeper = Sleeper()
self.filesystem_monitor_thread = threading.Thread(target=self.__filesystem_monitor)
self.filesystem_monitor_thread.start()
@@ -931,7 +932,8 @@ class DistributedObjectStore(ObjectStore):
def shutdown(self):
super(DistributedObjectStore, self).shutdown()
self.sleeper.wake()
if self.sleeper is not None:
self.sleeper.wake()
def exists(self, obj, **kwargs):
return self.__call_method('exists', obj, False, False, **kwargs)
lib/galaxy/security/validate_user_input.py
+6 -2
View File
@@ -23,8 +23,12 @@ def validate_publicname( trans, publicname, user=None ):
return ''
if user and user.username == publicname:
return ''
if len( publicname ) < 4:
return "Public name must be at least 4 characters in length"
if trans.webapp.name == 'tool_shed':
if len( publicname ) < 3:
return "Public name must be at least 3 characters in length"
else:
if len( publicname ) < 4:
return "Public name must be at least 4 characters in length"
if len( publicname ) > 255:
return "Public name cannot be more than 255 characters in length"
if not( VALID_PUBLICNAME_RE.match( publicname ) ):
lib/galaxy/tools/__init__.py
+76 -71
View File
@@ -48,6 +48,7 @@ from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repea
from galaxy.tools.parameters.input_translation import ToolInputTranslator
from galaxy.tools.parameters.output import ToolOutputActionGroup
from galaxy.tools.parameters.validation import LateValidationError
from galaxy.tools.filters import FilterFactory
from galaxy.tools.test import ToolTestBuilder
from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object, xml_text, xml_to_string
from galaxy.util.bunch import Bunch
@@ -120,6 +121,7 @@ class ToolBox( object ):
# (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag.
self.tool_root_dir = tool_root_dir
self.app = app
self.filter_factory = FilterFactory( self )
self.init_dependency_manager()
config_filenames = listify( config_filenames )
for config_filename in config_filenames:
@@ -680,76 +682,19 @@ class ToolBox( object ):
"""
return self.app.model.context
def to_dict( self, trans, in_panel=True, trackster=False ):
def filter_for_panel( item, filters ):
"""
Filters tool panel elements so that only those that are compatible
with provided filters are kept.
"""
def _apply_filter( filter_item, filter_list ):
for filter_method in filter_list:
if not filter_method( filter_item ):
return False
return True
if isinstance( item, Tool ):
if _apply_filter( item, filters[ 'tool' ] ):
return item
elif isinstance( item, ToolSectionLabel ):
if _apply_filter( item, filters[ 'label' ] ):
return item
elif isinstance( item, ToolSection ):
# Filter section item-by-item. Only show a label if there are
# non-filtered tools below it.
if _apply_filter( item, filters[ 'section' ] ):
cur_label_key = None
tools_under_label = False
filtered_elems = item.elems.copy()
for key, section_item in item.elems.items():
if isinstance( section_item, Tool ):
# Filter tool.
if _apply_filter( section_item, filters[ 'tool' ] ):
tools_under_label = True
else:
del filtered_elems[ key ]
elif isinstance( section_item, ToolSectionLabel ):
# If there is a label and it does not have tools,
# remove it.
if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ):
del filtered_elems[ cur_label_key ]
# Reset attributes for new label.
cur_label_key = key
tools_under_label = False
# Handle last label.
if cur_label_key and not tools_under_label:
del filtered_elems[ cur_label_key ]
# Only return section if there are elements.
if len( filtered_elems ) != 0:
copy = item.copy()
copy.elems = filtered_elems
return copy
return None
def to_dict( self, trans, in_panel=True, **kwds ):
#
# Dictify toolbox.
#
context = Bunch( toolbox=self, trans=trans, **kwds )
if in_panel:
panel_elts = [ val for val in self.tool_panel.itervalues() ]
# Filter if necessary.
filters = dict( tool=[ lambda x: not x._is_hidden_for_user( trans.user ) ], section=[], label=[] ) #hidden tools filter
if trackster:
filters[ 'tool' ].append( lambda x: x.trackster_conf ) # If tool has a trackster config, it can be used in Trackster.
filters = self.filter_factory.build_filters( trans, **kwds )
filtered_panel_elts = []
for index, elt in enumerate( panel_elts ):
elt = filter_for_panel( elt, filters )
elt = _filter_for_panel( elt, filters, context )
if elt:
filtered_panel_elts.append( elt )
panel_elts = filtered_panel_elts
@@ -760,13 +705,68 @@ class ToolBox( object ):
rval.append( elt.to_dict( trans, for_link=True ) )
else:
tools = []
for id, tool in self.app.toolbox.tools_by_id.items():
for id, tool in self.toolbox.tools_by_id.items():
tools.append( tool.to_dict( trans ) )
rval = tools
return rval
def _filter_for_panel( item, filters, context ):
"""
Filters tool panel elements so that only those that are compatible
with provided filters are kept.
"""
def _apply_filter( filter_item, filter_list ):
for filter_method in filter_list:
if not filter_method( context, filter_item ):
return False
return True
if isinstance( item, Tool ):
if _apply_filter( item, filters[ 'tool' ] ):
return item
elif isinstance( item, ToolSectionLabel ):
if _apply_filter( item, filters[ 'label' ] ):
return item
elif isinstance( item, ToolSection ):
# Filter section item-by-item. Only show a label if there are
# non-filtered tools below it.
if _apply_filter( item, filters[ 'section' ] ):
cur_label_key = None
tools_under_label = False
filtered_elems = item.elems.copy()
for key, section_item in item.elems.items():
if isinstance( section_item, Tool ):
# Filter tool.
if _apply_filter( section_item, filters[ 'tool' ] ):
tools_under_label = True
else:
del filtered_elems[ key ]
elif isinstance( section_item, ToolSectionLabel ):
# If there is a label and it does not have tools,
# remove it.
if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ):
del filtered_elems[ cur_label_key ]
# Reset attributes for new label.
cur_label_key = key
tools_under_label = False
# Handle last label.
if cur_label_key and not tools_under_label:
del filtered_elems[ cur_label_key ]
# Only return section if there are elements.
if len( filtered_elems ) != 0:
copy = item.copy()
copy.elems = filtered_elems
return copy
return None
class ToolSection( object ):
"""
A group of tools with similar type/purpose that will be displayed as a
@@ -816,6 +816,7 @@ class DefaultToolState( object ):
"""
def __init__( self ):
self.page = 0
self.rerun_remap_job_id = None
self.inputs = None
def encode( self, tool, app, secure=True ):
"""
@@ -825,6 +826,7 @@ class DefaultToolState( object ):
# page in that dict
value = params_to_strings( tool.inputs, self.inputs, app )
value["__page__"] = self.page
value["__rerun_remap_job_id__"] = self.rerun_remap_job_id
value = simplejson.dumps( value )
# Make it secure
if secure:
@@ -846,6 +848,10 @@ class DefaultToolState( object ):
# Restore from string
values = json_fix( simplejson.loads( value ) )
self.page = values.pop( "__page__" )
if '__rerun_remap_job_id__' in values:
self.rerun_remap_job_id = values.pop( "__rerun_remap_job_id__" )
else:
self.rerun_remap_job_id = None
self.inputs = params_from_strings( tool.inputs, values, app, ignore_errors=True )
class ToolOutput( object ):
@@ -933,6 +939,7 @@ class Tool( object ):
self.input_required = False
self.display_interface = True
self.require_login = False
self.rerun = False
# Define a place to keep track of all input These
# differ from the inputs dictionary in that inputs can be page
# elements like conditionals, but input_params are basic form
@@ -1521,7 +1528,8 @@ class Tool( object ):
elif ( re.search( "fatal", err_level, re.IGNORECASE ) ):
return_level = StdioErrorLevel.FATAL
else:
log.debug( "Error level %s did not match warning/fatal" % err_level )
log.debug( "Tool %s: error level %s did not match log/warning/fatal" %
( self.id, err_level ) )
except Exception:
log.error( "Exception in parse_error_level "
+ str(sys.exc_info() ) )
@@ -1933,7 +1941,10 @@ class Tool( object ):
# If we've completed the last page we can execute the tool
elif state.page == self.last_page:
try:
_, out_data = self.execute( trans, incoming=params, history=history )
rerun_remap_job_id = None
if 'rerun_remap_job_id' in incoming:
rerun_remap_job_id = trans.app.security.decode_id(incoming['rerun_remap_job_id'])
_, out_data = self.execute( trans, incoming=params, history=history, rerun_remap_job_id=rerun_remap_job_id )
except httpexceptions.HTTPFound, e:
#if it's a paste redirect exception, pass it up the stack
raise e
@@ -2502,7 +2513,6 @@ class Tool( object ):
datatypes_registry = self.app.datatypes_registry,
tool = self,
name = name )
if data:
for child in data.children:
param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child )
@@ -2914,11 +2924,6 @@ class Tool( object ):
self.sa_session.flush()
return primary_datasets
def _is_hidden_for_user( self, user ):
if self.hidden or ( not user and self.require_login ):
return True
return False
def to_dict( self, trans, for_link=False, for_display=False ):
""" Returns dict of tool. """
lib/galaxy/tools/actions/__init__.py
+57 -18
View File
@@ -1,15 +1,16 @@
from galaxy.model import LibraryDatasetDatasetAssociation
from galaxy.util.bunch import Bunch
from galaxy.util.odict import odict
from galaxy.util.json import to_json_string
from galaxy.tools.parameters import *
from galaxy.tools.parameters.grouping import *
from galaxy.util.template import fill_template
from galaxy.util.none_like import NoneDataset
from galaxy.web import url_for
from galaxy.exceptions import ObjectInvalid
import os
import galaxy.tools
from types import *
from galaxy.exceptions import ObjectInvalid
from galaxy.model import LibraryDatasetDatasetAssociation
from galaxy.tools.parameters import DataToolParameter, SelectToolParameter
from galaxy.tools.parameters.grouping import Conditional, Repeat
from galaxy.util.json import from_json_string
from galaxy.util.json import to_json_string
from galaxy.util.none_like import NoneDataset
from galaxy.util.odict import odict
from galaxy.util.template import fill_template
from galaxy.web import url_for
import logging
log = logging.getLogger( __name__ )
@@ -34,9 +35,11 @@ class DefaultToolAction( object ):
input_datasets = dict()
def visitor( prefix, input, value, parent = None ):
def process_dataset( data, formats = None ):
if not data:
return data
if formats is None:
formats = input.formats
if data and not isinstance( data.datatype, formats ):
if not data.datatype.matches_any( formats ):
# Need to refresh in case this conversion just took place, i.e. input above in tool performed the same conversion
trans.sa_session.refresh( data )
target_ext, converted_dataset = data.find_conversion_destination( formats )
@@ -54,7 +57,7 @@ class DefaultToolAction( object ):
trans.sa_session.flush()
data = new_data
current_user_roles = trans.get_current_user_roles()
if data and not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
if not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
raise "User does not have permission to use a dataset (%s) provided for input." % data.id
return data
if isinstance( input, DataToolParameter ):
@@ -70,7 +73,7 @@ class DefaultToolAction( object ):
conversions = []
for conversion_name, conversion_extensions, conversion_datatypes in input.conversions:
new_data = process_dataset( input_datasets[ prefix + input.name + str( i + 1 ) ], conversion_datatypes )
if not new_data or isinstance( new_data.datatype, conversion_datatypes ):
if not new_data or new_data.datatype.matches_any( conversion_datatypes ):
input_datasets[ prefix + conversion_name + str( i + 1 ) ] = new_data
conversions.append( ( conversion_name, new_data ) )
else:
@@ -90,7 +93,7 @@ class DefaultToolAction( object ):
conversions = []
for conversion_name, conversion_extensions, conversion_datatypes in input.conversions:
new_data = process_dataset( input_datasets[ prefix + input.name ], conversion_datatypes )
if not new_data or isinstance( new_data.datatype, conversion_datatypes ):
if not new_data or new_data.datatype.matches_any( conversion_datatypes ):
input_datasets[ prefix + conversion_name ] = new_data
conversions.append( ( conversion_name, new_data ) )
else:
@@ -105,7 +108,7 @@ class DefaultToolAction( object ):
tool.visit_inputs( param_values, visitor )
return input_datasets
def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, set_output_history=True, history=None, job_params=None ):
def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, set_output_history=True, history=None, job_params=None, rerun_remap_job_id=None):
"""
Executes a tool, creating job and tool outputs, associating them, and
submitting the job to the job queue. If history is not specified, use
@@ -214,7 +217,7 @@ class DefaultToolAction( object ):
if not chrom_info:
# Default to built-in build.
chrom_info = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey )
chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % input_dbkey )
incoming[ "chromInfo" ] = chrom_info
inp_data.update( db_datasets )
@@ -271,7 +274,9 @@ class DefaultToolAction( object ):
ext = input_ext
if output.format_source is not None and output.format_source in inp_data:
try:
ext = inp_data[output.format_source].ext
input_dataset = inp_data[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception, e:
pass
@@ -405,6 +410,40 @@ class DefaultToolAction( object ):
job.params = to_json_string( job_params )
job.set_handler(tool.get_job_handler(job_params))
trans.sa_session.add( job )
# Now that we have a job id, we can remap any outputs if this is a rerun and the user chose to continue dependent jobs
# This functionality requires tracking jobs in the database.
if trans.app.config.track_jobs_in_database and rerun_remap_job_id is not None:
try:
old_job = trans.sa_session.query( trans.app.model.Job ).get(rerun_remap_job_id)
assert old_job is not None, '(%s/%s): Old job id is invalid' % (rerun_remap_job_id, job.id)
assert old_job.tool_id == job.tool_id, '(%s/%s): Old tool id (%s) does not match rerun tool id (%s)' % (old_job.id, job.id, old_job.tool_id, job.tool_id)
if trans.user is not None:
assert old_job.user_id == trans.user.id, '(%s/%s): Old user id (%s) does not match rerun user id (%s)' % (old_job.id, job.id, old_job.user_id, trans.user.id)
elif trans.user is None and type( galaxy_session ) == trans.model.GalaxySession:
assert old_job.session_id == galaxy_session.id, '(%s/%s): Old session id (%s) does not match rerun session id (%s)' % (old_job.id, job.id, old_job.session_id, galaxy_session.id)
else:
raise Exception('(%s/%s): Remapping via the API is not (yet) supported' % (old_job.id, job.id))
for jtod in old_job.output_datasets:
for (job_to_remap, jtid) in [(jtid.job, jtid) for jtid in jtod.dataset.dependent_jobs]:
if (trans.user is not None and job_to_remap.user_id == trans.user.id) or (trans.user is None and job_to_remap.session_id == galaxy_session.id):
if job_to_remap.state == job_to_remap.states.PAUSED:
job_to_remap.state = job_to_remap.states.NEW
for hda in [ dep_jtod.dataset for dep_jtod in job_to_remap.output_datasets ]:
if hda.state == hda.states.PAUSED:
hda.state = hda.states.NEW
hda.info = None
for p in job_to_remap.parameters:
if p.name == jtid.name and p.value == str(jtod.dataset.id):
p.value = str(out_data[jtod.name].id)
jtid.dataset = out_data[jtod.name]
jtid.dataset.hid = jtod.dataset.hid
log.info('Job %s input HDA %s remapped to new HDA %s' % (job_to_remap.id, jtod.dataset.id, jtid.dataset.id))
trans.sa_session.add(job_to_remap)
trans.sa_session.add(jtid)
jtod.dataset.visible = False
trans.sa_session.add(jtod)
except Exception, e:
log.exception('Cannot remap rerun dependencies.')
trans.sa_session.flush()
# Some tools are not really executable, but jobs are still created for them ( for record keeping ).
# Examples include tools that redirect to other applications ( epigraph ). These special tools must
lib/galaxy/tools/actions/history_imp_exp.py
+2 -2
View File
@@ -9,7 +9,7 @@ log = logging.getLogger( __name__ )
class ImportHistoryToolAction( ToolAction ):
"""Tool action used for importing a history to an archive. """
def execute( self, tool, trans, incoming = {}, set_output_hid = False, overwrite = True, history=None ):
def execute( self, tool, trans, incoming = {}, set_output_hid = False, overwrite = True, history=None, **kwargs ):
#
# Create job.
#
@@ -57,7 +57,7 @@ class ImportHistoryToolAction( ToolAction ):
class ExportHistoryToolAction( ToolAction ):
"""Tool action used for exporting a history to an archive. """
def execute( self, tool, trans, incoming = {}, set_output_hid = False, overwrite = True, history=None ):
def execute( self, tool, trans, incoming = {}, set_output_hid = False, overwrite = True, history=None, **kwargs ):
#
# Get history to export.
#
lib/galaxy/tools/actions/metadata.py
+1 -1
View File
@@ -9,7 +9,7 @@ log = logging.getLogger( __name__ )
class SetMetadataToolAction( ToolAction ):
"""Tool action used for setting external metadata on an existing dataset"""
def execute( self, tool, trans, incoming={}, set_output_hid=False, overwrite=True, history=None, job_params=None ):
def execute( self, tool, trans, incoming={}, set_output_hid=False, overwrite=True, history=None, job_params=None, **kwargs ):
"""
Execute using a web transaction.
"""
lib/galaxy/tools/actions/upload.py
+1 -1
View File
@@ -5,7 +5,7 @@ import logging
log = logging.getLogger( __name__ )
class UploadToolAction( ToolAction ):
def execute( self, tool, trans, incoming={}, set_output_hid = True, history=None ):
def execute( self, tool, trans, incoming={}, set_output_hid = True, history=None, **kwargs ):
dataset_upload_inputs = []
for input_name, input in tool.inputs.iteritems():
if input.type == "upload_dataset":
lib/galaxy/tools/data/__init__.py
+155 -93
View File
@@ -13,9 +13,11 @@ import shutil
import tempfile
from galaxy import util
from galaxy.util.odict import odict
log = logging.getLogger( __name__ )
DEFAULT_TABLE_TYPE = 'tabular'
class ToolDataTableManager( object ):
"""Manages a collection of tool data tables"""
@@ -26,9 +28,6 @@ class ToolDataTableManager( object ):
# at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries
# from that file are inserted into this dict at the time of installation.
self.data_tables = {}
# Store config elements for on-the-fly persistence to the defined shed_tool_data_table_config file name.
self.shed_data_table_elems = []
self.data_table_elem_names = []
if config_filename:
self.load_from_config_file( config_filename, self.tool_data_path, from_shed_config=False )
@@ -58,23 +57,15 @@ class ToolDataTableManager( object ):
root = tree.getroot()
table_elems = []
for table_elem in root.findall( 'table' ):
type = table_elem.get( 'type', 'tabular' )
assert type in tool_data_table_types, "Unknown data table type '%s'" % type
table = ToolDataTable.from_elem( table_elem, tool_data_path, from_shed_config )
table_elems.append( table_elem )
table_elem_name = table_elem.get( 'name', None )
if table_elem_name and table_elem_name not in self.data_table_elem_names:
self.data_table_elem_names.append( table_elem_name )
if from_shed_config:
self.shed_data_table_elems.append( table_elem )
table = tool_data_table_types[ type ]( table_elem, tool_data_path, from_shed_config)
if table.name not in self.data_tables:
self.data_tables[ table.name ] = table
log.debug( "Loaded tool data table '%s'", table.name )
else:
for table_row in table.data:
# FIXME: This does not account for an entry with the same unique build ID, but a different path.
if table_row not in self.data_tables[ table.name ].data:
self.data_tables[ table.name ].data.append( table_row )
log.debug( "Loading another instance of data table '%s', attempting to merge content.", table.name )
self.data_tables[ table.name ].merge_tool_data_table( table, allow_duplicates=False ) #only merge content, do not persist to disk, do not allow duplicate rows when merging
# FIXME: This does not account for an entry with the same unique build ID, but a different path.
return table_elems
def add_new_entries_from_config_file( self, config_filename, tool_data_path, shed_tool_data_table_config, persist=False ):
@@ -100,84 +91,98 @@ class ToolDataTableManager( object ):
"""
error_message = ''
table_elems = []
try:
tree = util.parse_xml( config_filename )
root = tree.getroot()
except Exception, e:
error_message = 'Error attempting to parse file %s: %s' % ( str( os.path.split( config_filename )[ 1 ] ), str( e ) )
log.debug( error_message )
return table_elems, error_message
# Make a copy of the current list of data_table_elem_names so we can persist later if changes to the config file are necessary.
original_data_table_elem_names = [ name for name in self.data_table_elem_names ]
if root.tag == 'tables':
table_elems = self.load_from_config_file( config_filename=config_filename,
tool_data_path=tool_data_path,
from_shed_config=True )
else:
type = root.get( 'type', 'tabular' )
assert type in tool_data_table_types, "Unknown data table type '%s'" % type
table_elems.append( root )
table_elem_name = root.get( 'name', None )
if table_elem_name and table_elem_name not in self.data_table_elem_names:
self.data_table_elem_names.append( table_elem_name )
self.shed_data_table_elems.append( root )
table = tool_data_table_types[ type ]( root, tool_data_path )
if table.name not in self.data_tables:
self.data_tables[ table.name ] = table
log.debug( "Added new tool data table '%s'", table.name )
if persist and self.data_table_elem_names != original_data_table_elem_names:
except Exception, e:
error_message = 'Error attempting to parse file %s: %s' % ( str( os.path.split( config_filename )[ 1 ] ), str( e ) )
log.debug( error_message )
table_elems = []
if persist:
# Persist Galaxy's version of the changed tool_data_table_conf.xml file.
self.to_xml_file( shed_tool_data_table_config )
self.to_xml_file( shed_tool_data_table_config, table_elems )
return table_elems, error_message
def to_xml_file( self, shed_tool_data_table_config ):
"""Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk."""
def to_xml_file( self, shed_tool_data_table_config, new_elems=None, remove_elems=None ):
"""
Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk.
remove_elems are removed before new_elems are added.
"""
if not ( new_elems or remove_elems ):
log.debug( 'ToolDataTableManager.to_xml_file called without any elements to add or remove.' )
return #no changes provided, no need to persist any changes
if not new_elems:
new_elems = []
if not remove_elems:
remove_elems = []
full_path = os.path.abspath( shed_tool_data_table_config )
fd, filename = tempfile.mkstemp()
os.write( fd, '<?xml version="1.0"?>\n' )
os.write( fd, '<tables>\n' )
for elem in self.shed_data_table_elems:
os.write( fd, '%s' % util.xml_to_string( elem ) )
os.write( fd, '</tables>\n' )
os.close( fd )
shutil.move( filename, full_path )
#FIXME: we should lock changing this file by other threads / head nodes
try:
tree = util.parse_xml( full_path )
root = tree.getroot()
out_elems = [ elem for elem in root ]
except Exception, e:
out_elems = []
log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', e )
for elem in remove_elems:
#handle multiple occurrences of remove elem in existing elems
while elem in out_elems:
remove_elems.remove( elem )
#add new elems
out_elems.extend( new_elems )
with open( full_path, 'wb' ) as out:
out.write( '<?xml version="1.0"?>\n<tables>\n' )
for elem in out_elems:
out.write( util.xml_to_string( elem ) )
out.write( '</tables>\n' )
os.chmod( full_path, 0644 )
class ToolDataTable( object ):
@classmethod
def from_elem( cls, table_elem, tool_data_path, from_shed_config ):
table_type = table_elem.get( 'type', 'tabular' )
assert table_type in tool_data_table_types, "Unknown data table type '%s'" % type
return tool_data_table_types[ table_type ]( table_elem, tool_data_path, from_shed_config=from_shed_config )
def __init__( self, config_element, tool_data_path, from_shed_config = False):
self.name = config_element.get( 'name' )
self.comment_char = config_element.get( 'comment_char' )
self.empty_field_value = config_element.get( 'empty_field_value', '' )
self.empty_field_values = {}
for file_elem in config_element.findall( 'file' ):
# There should only be one file_elem.
if 'path' in file_elem.attrib:
tool_data_file_path = file_elem.get( 'path' )
self.tool_data_file = os.path.split( tool_data_file_path )[1]
else:
self.tool_data_file = None
self.filenames = odict()
self.tool_data_path = tool_data_path
self.missing_index_file = None
# increment this variable any time a new entry is added, or when the table is totally reloaded
# This value has no external meaning, and does not represent an abstract version of the underlying data
self._loaded_content_version = 1
def _update_version( self ):
self._loaded_content_version += 1
return self._loaded_content_version
def get_empty_field_by_name( self, name ):
return self.empty_field_values.get( name, self.empty_field_value )
def _add_entry( self, entry, persist=False, persist_on_error=False, **kwd ):
def _add_entry( self, entry, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ):
raise NotImplementedError( "Abstract method" )
def add_entry( self, entry, persist=False, persist_on_error=False, **kwd ):
self._add_entry( entry, persist=persist, persist_on_error=persist_on_error, **kwd )
self._loaded_content_version += 1
def add_entry( self, entry, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ):
self._add_entry( entry, allow_duplicates=allow_duplicates, persist=persist, persist_on_error=persist_on_error, entry_source=entry_source, **kwd )
return self._update_version()
def add_entries( self, entries, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ):
if entries:
for entry in entries:
self.add_entry( entry, allow_duplicates=allow_duplicates, persist=persist, persist_on_error=persist_on_error, entry_source=entry_source, **kwd )
return self._loaded_content_version
def is_current_version( self, other_version ):
return self._loaded_content_version == other_version
def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ):
raise NotImplementedError( "Abstract method" )
class TabularToolDataTable( ToolDataTable ):
"""
@@ -196,6 +201,7 @@ class TabularToolDataTable( ToolDataTable ):
def __init__( self, config_element, tool_data_path, from_shed_config = False):
super( TabularToolDataTable, self ).__init__( config_element, tool_data_path, from_shed_config)
self.data = []
self.configure_and_load( config_element, tool_data_path, from_shed_config)
def configure_and_load( self, config_element, tool_data_path, from_shed_config = False):
@@ -206,24 +212,37 @@ class TabularToolDataTable( ToolDataTable ):
self.comment_char = config_element.get( 'comment_char', '#' )
# Configure columns
self.parse_column_spec( config_element )
#store repo info if available:
repo_elem = config_element.find( 'tool_shed_repository' )
if repo_elem is not None:
repo_info = dict( tool_shed=repo_elem.find( 'tool_shed' ).text, name=repo_elem.find( 'repository_name' ).text,
owner=repo_elem.find( 'repository_owner' ).text, installed_changeset_revision=repo_elem.find( 'installed_changeset_revision' ).text )
else:
repo_info = None
# Read every file
all_rows = []
for file_element in config_element.findall( 'file' ):
filename = file_path = file_element.get( 'path', None )
found = False
if file_path is None:
log.debug( "Encountered a file element (%s) that does not contain a path value when loading tool data table '%s'.", util.xml_to_string( file_element ), self.name )
continue
#FIXME: splitting on and merging paths from a configuration file when loading is wonky
# Data should exist on disk in the state needed, i.e. the xml configuration should
# point directly to the desired file to load. Munging of the tool_data_tables_conf.xml.sample
# can be done during installing / testing / metadata resetting with the creation of a proper
# tool_data_tables_conf.xml file, containing correct <file path=> attributes. Allowing a
# path.join with a different root should be allowed, but splitting should not be necessary.
if tool_data_path and from_shed_config:
# Must identify with from_shed_config as well, because the
# regular galaxy app has and uses tool_data_path.
# We're loading a tool in the tool shed, so we cannot use the Galaxy tool-data
# directory which is hard-coded into the tool_data_table_conf.xml entries.
filepath = file_element.get( 'path' )
filename = os.path.split( filepath )[ 1 ]
filename = os.path.split( file_path )[ 1 ]
filename = os.path.join( tool_data_path, filename )
else:
filename = file_element.get( 'path' )
if os.path.exists( filename ):
found = True
all_rows.extend( self.parse_file_fields( open( filename ) ) )
self.filename = filename
else:
# Since the path attribute can include a hard-coded path to a specific directory
# (e.g., <file path="tool-data/cg_crr_files.loc" />) which may not be the same value
@@ -233,14 +252,32 @@ class TabularToolDataTable( ToolDataTable ):
if file_path and file_path != self.tool_data_path:
corrected_filename = os.path.join( self.tool_data_path, file_name )
if os.path.exists( corrected_filename ):
filename = corrected_filename
found = True
all_rows.extend( self.parse_file_fields( open( corrected_filename ) ) )
self.filename = corrected_filename
if not found:
if found:
self.data.extend( self.parse_file_fields( open( filename ) ) )
self._update_version()
else:
self.missing_index_file = filename
log.warn( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) )
self.data = all_rows
if filename not in self.filenames or not self.filenames[ filename ][ 'found' ]:
self.filenames[ filename ] = dict( found=found, filename=filename, from_shed_config=from_shed_config, tool_data_path=tool_data_path,
config_element=config_element, tool_shed_repository=repo_info )
else:
log.debug( "Filename '%s' already exists in filenames (%s), not adding", filename, self.filenames.keys() )
def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ):
assert self.columns == other_table.columns, "Merging tabular data tables with non matching columns is not allowed: %s:%s != %s:%s" % ( self.name, self.columns, other_table.name, other_table.columns )
#merge filename info
for filename, info in other_table.filenames.iteritems():
if filename not in self.filenames:
self.filenames[ filename ] = info
#add data entries and return current data table version
return self.add_entries( other_table.data, allow_duplicates=allow_duplicates, persist=persist, persist_on_error=persist_on_error, entry_source=entry_source, **kwd )
def handle_found_index_file( self, filename ):
self.missing_index_file = None
self.data.extend( self.parse_file_fields( open( filename ) ) )
@@ -341,7 +378,7 @@ class TabularToolDataTable( ToolDataTable ):
break
return rval
def _add_entry( self, entry, persist=False, persist_on_error=False, **kwd ):
def _add_entry( self, entry, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ):
#accepts dict or list of columns
if isinstance( entry, dict ):
fields = []
@@ -354,28 +391,53 @@ class TabularToolDataTable( ToolDataTable ):
fields.append( field_value )
else:
fields = entry
is_error = False
if self.largest_index < len( fields ):
fields = self._replace_field_separators( fields )
self.data.append( fields )
field_len_error = False
if fields not in self.data or allow_duplicates:
self.data.append( fields )
else:
log.error( "Attempted to add fields (%s) to data table '%s', but this entry already exists and allow_duplicates is False.", fields, self.name )
is_error = True
else:
log.error( "Attempted to add fields (%s) to data table '%s', but there were not enough fields specified ( %i < %i ).", fields, self.name, len( fields ), self.largest_index + 1 )
field_len_error = True
if persist and ( not field_len_error or persist_on_error ):
#FIXME: Need to lock these files for editing
try:
data_table_fh = open( self.filename, 'r+b' )
except IOError, e:
log.warning( 'Error opening data table file (%s) with r+b, assuming file does not exist and will open as wb: %s', self.filename, e )
data_table_fh = open( self.filename, 'wb' )
if os.stat( self.filename )[6] != 0:
# ensure last existing line ends with new line
data_table_fh.seek( -1, 2 ) #last char in file
last_char = data_table_fh.read( 1 )
if last_char not in [ '\n', '\r' ]:
data_table_fh.write( '\n' )
data_table_fh.write( "%s\n" % ( self.separator.join( fields ) ) )
return not field_len_error
is_error = True
filename = None
if persist and ( not is_error or persist_on_error ):
if entry_source:
#if dict, assume is compatible info dict, otherwise call method
if isinstance( entry_source, dict ):
source_repo_info = entry_source
else:
source_repo_info = entry_source.get_tool_shed_repository_info_dict()
else:
source_repo_info = None
for name, value in self.filenames.iteritems():
repo_info = value.get( 'tool_shed_repository', None )
if ( not source_repo_info and not repo_info ) or ( source_repo_info and repo_info and source_repo_info == repo_info ):
filename = name
break
if filename is None:
#should we default to using any filename here instead?
log.error( "Unable to determine filename for persisting data table '%s' values: '%s'.", self.name, fields )
is_error = True
else:
#FIXME: Need to lock these files for editing
log.debug( "Persisting changes to file: %s", filename )
try:
data_table_fh = open( filename, 'r+b' )
except IOError, e:
log.warning( 'Error opening data table file (%s) with r+b, assuming file does not exist and will open as wb: %s', self.filename, e )
data_table_fh = open( filename, 'wb' )
if os.stat( filename )[6] != 0:
# ensure last existing line ends with new line
data_table_fh.seek( -1, 2 ) #last char in file
last_char = data_table_fh.read( 1 )
if last_char not in [ '\n', '\r' ]:
data_table_fh.write( '\n' )
data_table_fh.write( "%s\n" % ( self.separator.join( fields ) ) )
return not is_error
def _replace_field_separators( self, fields, separator=None, replace=None, comment_char=None ):
#make sure none of the fields contain separator
lib/galaxy/tools/data_manager/manager.py
+33 -12
View File
@@ -16,6 +16,8 @@ import logging
log = logging.getLogger( __name__ )
SUPPORTED_DATA_TABLE_TYPES = ( TabularToolDataTable )
VALUE_TRANSLATION_FUNCTIONS = dict( abspath=os.path.abspath )
DEFAULT_VALUE_TRANSLATION_TYPE = 'template'
class DataManagers( object ):
def __init__( self, app, xml_filename=None ):
@@ -106,6 +108,7 @@ class DataManager( object ):
self.output_ref_by_data_table = {}
self.move_by_data_table_column = {}
self.value_translation_by_data_table_column = {}
self.tool_shed_repository_info_dict = None
if elem is not None:
self.load_from_element( elem, tool_path or self.data_managers.tool_path )
def load_from_element( self, elem, tool_path ):
@@ -126,6 +129,9 @@ class DataManager( object ):
repository_name = tool_elem.find( 'repository_name' ).text
repository_owner = tool_elem.find( 'repository_owner' ).text
installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text
#save repository info here
self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision )
#get tool_shed repo id
tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.data_managers.app, tool_shed, repository_name, repository_owner, installed_changeset_revision )
tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id )
#use shed_conf_file to determine tool_path
@@ -156,15 +162,24 @@ class DataManager( object ):
if data_table_name not in self.output_ref_by_data_table:
self.output_ref_by_data_table[ data_table_name ] = {}
self.output_ref_by_data_table[ data_table_name ][ data_table_coumn_name ] = output_ref
value_translation_elem = column_elem.find( 'value_translation' )
if value_translation_elem is not None:
value_translation = value_translation_elem.text
else:
value_translation = None
if value_translation is not None:
if data_table_name not in self.value_translation_by_data_table_column:
self.value_translation_by_data_table_column[ data_table_name ] = {}
self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = value_translation
value_translation_elems = column_elem.findall( 'value_translation' )
if value_translation_elems is not None:
for value_translation_elem in value_translation_elems:
value_translation = value_translation_elem.text
if value_translation is not None:
value_translation_type = value_translation_elem.get( 'type', DEFAULT_VALUE_TRANSLATION_TYPE )
if data_table_name not in self.value_translation_by_data_table_column:
self.value_translation_by_data_table_column[ data_table_name ] = {}
if data_table_coumn_name not in self.value_translation_by_data_table_column[ data_table_name ]:
self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = []
if value_translation_type == 'function':
if value_translation in VALUE_TRANSLATION_FUNCTIONS:
value_translation = VALUE_TRANSLATION_FUNCTIONS[ value_translation ]
else:
raise ValueError( "Unsupported value translation function: '%s'" % ( value_translation ) )
else:
assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError( "Unsupported value translation type: '%s'" % ( value_translation_type ) )
self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ].append( value_translation )
for move_elem in column_elem.findall( 'move' ):
move_type = move_elem.get( 'type', 'directory' )
@@ -241,7 +256,7 @@ class DataManager( object ):
if name in output_ref_values:
moved = self.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value )
data_table_value[ name ] = self.process_value_translation( data_table_name, name, **data_table_value )
data_table.add_entry( data_table_value, persist=True )
data_table.add_entry( data_table_value, persist=True, entry_source=self )
for data_table_name, data_table_values in data_tables_dict.iteritems():
#tool returned extra data table entries, but data table was not declared in data manager
@@ -286,6 +301,12 @@ class DataManager( object ):
def process_value_translation( self, data_table_name, column_name, **kwd ):
value = kwd.get( column_name )
if data_table_name in self.value_translation_by_data_table_column and column_name in self.value_translation_by_data_table_column[ data_table_name ]:
value_translation = self.value_translation_by_data_table_column[ data_table_name ][ column_name ]
value = fill_template( value_translation, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd )
for value_translation in self.value_translation_by_data_table_column[ data_table_name ][ column_name ]:
if isinstance( value_translation, basestring ):
value = fill_template( value_translation, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd )
else:
value = value_translation( value )
return value
def get_tool_shed_repository_info_dict( self ):
return self.tool_shed_repository_info_dict
lib/galaxy/tools/filters/__init__.py
+67
View File
@@ -0,0 +1,67 @@
class FilterFactory( object ):
"""
An instance of this class is responsible for filtering the list
of tools presented to a given user in a given context.
"""
def __init__( self, toolbox ):
self.toolbox = toolbox
# Prepopulate dict containing filters that are always checked,
# other filters that get checked depending on context (e.g. coming from
# trackster or no user found are added in build filters).
self.default_filters = dict( tool=[ _not_hidden ], section=[], label=[] )
# Add dynamic filters to these default filters.
config = toolbox.app.config
self.__init_filters( "tool", config.tool_filters )
self.__init_filters( "section", config.tool_section_filters )
self.__init_filters( "label", config.tool_label_filters )
def build_filters( self, trans, **kwds ):
"""
Build list of filters to check tools against given current context.
"""
filters = self.default_filters.copy()
if not trans.user:
filters[ "tool" ].append( _requires_login )
if kwds.get( "trackster", False ):
filters[ "tool" ].append( _has_trackster_conf )
return filters
def __init_filters( self, key, filters ):
for filter in filters:
filter_function = self.__build_filter_function( filter )
self.default_filters[ key ].append( filter_function )
def __build_filter_function( self, filter_name ):
"""Obtain python function (importing a submodule if needed)
corresponding to filter_name.
"""
if ":" in filter_name:
# Should be a submodule of filters (e.g. examples:restrict_development_tools)
(module_name, function_name) = filter_name.rsplit(":", 1)
module = __import__( module_name.strip(), globals() )
function = getattr( module, function_name.strip() )
else:
# No module found, just load a function from this file or
# one that has be explicitly imported.
function = getattr( globals(), filter_name.strip() )
return function
## Stock Filter Functions
def _not_hidden( context, tool ):
return not tool.hidden
def _requires_login( context, tool ):
return not tool.require_login
def _has_trackster_conf( context, tool ):
return tool.trackster_conf
lib/galaxy/tools/filters/examples.py.sample
+87
View File
@@ -0,0 +1,87 @@
import logging
log = logging.getLogger( __name__ )
def restrict_upload_to_admins( context, tool ):
"""
This tool filter will hide the upload tool from all users except admin
users. This can be enabled by renaming this file to examples.py and adding
the following to the ``app:main`` section of ``universe_wsgi.ini``:
tool_filters = examples:restrict_upload_to_admins
"""
if tool.name == "Upload File":
return context.trans.user_is_admin()
return True
def disable_gatk( context, tool ):
"""
This tool filter will disable all gatk tools when enabled. This can be
enabled by renaming this file to examples.py and adding the following to the
``app:main`` section of ``universe_wsgi.ini``:
tool_filters = examples:disable_gatk
"""
return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] )
def explicit_user_mapping( context, section ):
"""
This tool section filter uses an explicit mapping to describe what users can view
which tool sections. Anonymous users will only be able to view the "Get Data"
tool section (with id getext). This can be enabled by renaming this file to
examples.py and adding the following to the ``app:main`` section of
``universe_wsgi.ini``:
tool_section_filters = examples:explicit_user_mapping
"""
users_sections = {
None: [ "getext" ],
"bob@example.com": [ "getext", "textutil", "filter" ],
"mary@example.com": [ "getext", "textutil", "filter", "ngs" ],
}
user = context.trans.user
email = user and user.email
valid_sections = users_sections.get( email, [] )
return section.id in valid_sections
DEVELOPERS = [ "mary@example.com" ]
def restrict_development_tools( context, tool ):
"""
This tool filter will disable all tools with the string alpha appearing in
the version for all users except those explicitly appearing in the DEVELOPERS list
defined above. This can be enabled by renaming this file to examples.py and
adding the following to the ``app:main`` section of ``universe_wsgi.ini``:
tool_filters = examples:restrict_development_tools
"""
version = tool.version
user = context.trans.user
email = user and user.email
return "alpha" not in version or email in DEVELOPERS
def per_host_tool_sections( context, section ):
"""
This tool section filter results in different sections being display based on
the URL the user is making the request to. This could allow a single Galaxy instance
to seem like several different instances hosting different tools based on the URL used
to access the Galxy. This can be enabled by renaming this file to examples.py and adding
the following to the ``app:main`` section of ``universe_wsgi.ini``:
tool_section_filters = examples:per_host_tool_sections
"""
host = context.trans.request.host
# Core tools used by all virtual hosts.
valid_sections = [ "getext", "textutil", "filter" ]
if "ngs.galaxy.example.com" in host:
valid_sections += [ "ngs" ]
elif "microarray.galaxy.example.com" in host:
valid_sections += [ "microarray" ]
elif "proteomics.galaxy.example.com" in host:
valid_sections += [ "proteomics" ]
return section.id in valid_sections
lib/galaxy/tools/loader.py
-196
View File
@@ -202,199 +202,3 @@ def _xml_replace(query, targets, parent_map):
current_index += 1
parent_el.insert(current_index, deepcopy(target))
parent_el.remove(query)
def test_loader():
"""
Function to test this module. Galaxy doesn't seem to have a
place to put unit tests that are not doctests. These tests can
be run with nosetests via the following command:
% nosetests lib/galaxy/tools/loader.py
"""
from tempfile import mkdtemp
from shutil import rmtree
class TestToolDirectory(object):
def __init__(self):
self.temp_directory = mkdtemp()
def __enter__(self):
return self
def __exit__(self, type, value, tb):
rmtree(self.temp_directory)
def write(self, contents, name="tool.xml"):
open(os.path.join(self.temp_directory, name), "w").write(contents)
def load(self, name="tool.xml", preprocess=True):
if preprocess:
loader = load_tool
else:
loader = parse_xml
return loader(os.path.join(self.temp_directory, name))
## Test simple macro replacement.
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<expand macro="inputs" />
<macros>
<macro name="inputs">
<inputs />
</macro>
</macros>
</tool>''')
xml = tool_dir.load(preprocess=False)
assert xml.find("inputs") is None
xml = tool_dir.load(preprocess=True)
assert xml.find("inputs") is not None
# Test importing macros from external files
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<expand macro="inputs" />
<macros>
<import>external.xml</import>
</macros>
</tool>''')
tool_dir.write('''
<macros>
<macro name="inputs">
<inputs />
</macro>
</macros>''', name="external.xml")
xml = tool_dir.load(preprocess=False)
assert xml.find("inputs") is None
xml = tool_dir.load(preprocess=True)
assert xml.find("inputs") is not None
# Test macros with unnamed yield statements.
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<expand macro="inputs">
<input name="first_input" />
</expand>
<macros>
<macro name="inputs">
<inputs>
<yield />
</inputs>
</macro>
</macros>
</tool>''')
xml = tool_dir.load()
assert xml.find("inputs").find("input").get("name") == "first_input"
# Test recursive macro applications.
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<expand macro="inputs">
<input name="first_input" />
<expand macro="second" />
</expand>
<macros>
<macro name="inputs">
<inputs>
<yield />
</inputs>
</macro>
<macro name="second">
<input name="second_input" />
</macro>
</macros>
</tool>''')
xml = tool_dir.load()
assert xml.find("inputs").findall("input")[1].get("name") == "second_input"
# Test recursive macro applications.
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<expand macro="inputs">
<input name="first_input" />
<expand macro="second" />
</expand>
<macros>
<macro name="inputs">
<inputs>
<yield />
</inputs>
</macro>
<macro name="second">
<expand macro="second_delegate" />
</macro>
<macro name="second_delegate">
<input name="second_input" />
</macro>
</macros>
</tool>''')
xml = tool_dir.load()
assert xml.find("inputs").findall("input")[1].get("name") == "second_input"
# Test <xml> is shortcut for macro type="xml"
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<expand macro="inputs" />
<macros>
<xml name="inputs">
<inputs />
</xml>
</macros>
</tool>''')
xml = tool_dir.load()
assert xml.find("inputs") is not None
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<command interpreter="python">tool_wrapper.py
#include source=$tool_params
</command>
<macros>
<template name="tool_params">-a 1 -b 2</template>
</macros>
</tool>
''')
xml = tool_dir.load()
params_dict = template_macro_params(xml.getroot())
assert params_dict['tool_params'] == "-a 1 -b 2"
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<macros>
<token name="@CITATION@">The citation.</token>
</macros>
<help>@CITATION@</help>
<another>
<tag />
</another>
</tool>
''')
xml = tool_dir.load()
help_el = xml.find("help")
assert help_el.text == "The citation.", help_el.text
with TestToolDirectory() as tool_dir:
tool_dir.write('''
<tool>
<macros>
<token name="@TAG_VAL@">The value.</token>
</macros>
<another>
<tag value="@TAG_VAL@" />
</another>
</tool>
''')
xml = tool_dir.load()
tag_el = xml.find("another").find("tag")
value = tag_el.get('value')
assert value == "The value.", value
lib/galaxy/tools/parameters/basic.py
+26 -22
View File
@@ -703,7 +703,10 @@ class SelectToolParameter( ToolParameter ):
value = None
else:
if not isinstance( value, list ):
value = value.split( "\n" )
# Split on all whitespace. This not only provides flexibility
# in interpreting values but also is needed because many browsers
# use \r\n to separate lines.
value = value.split()
return UnvalidatedValue( value )
legal_values = self.get_legal_values( trans, context )
if isinstance( value, list ):
@@ -801,7 +804,7 @@ class SelectToolParameter( ToolParameter ):
if isinstance( dep_value, RuntimeValue ):
return True
#dataset not ready yet
if hasattr( self, 'ref_input' ) and isinstance( dep_value, self.tool.app.model.HistoryDatasetAssociation ) and ( dep_value.is_pending or not isinstance( dep_value.datatype, self.ref_input.formats ) ):
if hasattr( self, 'ref_input' ) and isinstance( dep_value, self.tool.app.model.HistoryDatasetAssociation ) and ( dep_value.is_pending or not dep_value.datatype.matches_any( self.ref_input.formats ) ):
return True
# Dynamic, but all dependenceis are known and have values
return False
@@ -1056,7 +1059,7 @@ class ColumnListParameter( SelectToolParameter ):
if not dataset.metadata.columns:
# Only allow late validation if the dataset is not yet ready
# (since we have reason to expect the metadata to be ready eventually)
if dataset.is_pending or not isinstance( dataset.datatype, self.ref_input.formats ):
if dataset.is_pending or not dataset.datatype.matches_any( self.ref_input.formats ):
return True
# No late validation
return False
@@ -1390,25 +1393,26 @@ class DataToolParameter( ToolParameter ):
# Add metadata validator
if not string_as_bool( elem.get( 'no_validation', False ) ):
self.validators.append( validation.MetadataValidator() )
# Find datatypes_registry
if tool is None:
if trans:
# Must account for "Input Dataset" types, which while not a tool still need access to the real registry.
# A handle to the transaction (and thus app) will be given by the module.
datatypes_registry = trans.app.datatypes_registry
else:
#This occurs for things such as unit tests
import galaxy.datatypes.registry
datatypes_registry = galaxy.datatypes.registry.Registry()
datatypes_registry.load_datatypes()
else:
datatypes_registry = tool.app.datatypes_registry
# Build tuple of classes for supported data formats
formats = []
self.extensions = elem.get( 'format', 'data' ).split( "," )
for extension in self.extensions:
extension = extension.strip()
if tool is None:
if trans:
# Must account for "Input Dataset" types, which while not a tool still need access to the real registry.
# A handle to the transaction (and thus app) will be given by the module.
formats.append( trans.app.datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ )
else:
#This occurs for things such as unit tests
import galaxy.datatypes.registry
datatypes_registry = galaxy.datatypes.registry.Registry()
datatypes_registry.load_datatypes()
formats.append( datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ )
else:
formats.append( tool.app.datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ )
self.formats = tuple( formats )
normalized_extensions = [extension.strip().lower() for extension in self.extensions]
for extension in normalized_extensions:
formats.append( datatypes_registry.get_datatype_by_extension( extension ) )
self.formats = formats
self.multiple = string_as_bool( elem.get( 'multiple', False ) )
# TODO: Enhance dynamic options for DataToolParameters. Currently,
# only the special case key='build' of type='data_meta' is
@@ -1432,7 +1436,7 @@ class DataToolParameter( ToolParameter ):
conv_extensions = conv_elem.get( "type" ) #target datatype extension
# FIXME: conv_extensions should be able to be an ordered list
assert None not in [ name, type ], 'A name (%s) and type (%s) are required for explicit conversion' % ( name, type )
conv_types = tool.app.datatypes_registry.get_datatype_by_extension( conv_extensions.lower() ).__class__
conv_types = tool.app.datatypes_registry.get_datatype_by_extension( conv_extensions.lower() )
self.conversions.append( ( name, conv_extensions, conv_types ) )
def get_html_field( self, trans=None, value=None, other_values={} ):
@@ -1467,7 +1471,7 @@ class DataToolParameter( ToolParameter ):
continue
if self.options and self._options_filter_attribute( hda ) != filter_value:
continue
if isinstance( hda.datatype, self.formats):
if hda.datatype.matches_any( self.formats ):
selected = ( value and ( hda in value ) )
if hda.visible:
hidden_text = ""
@@ -1534,7 +1538,7 @@ class DataToolParameter( ToolParameter ):
for i, data in enumerate( datasets ):
if data.visible and not data.deleted and data.state not in [data.states.ERROR, data.states.DISCARDED]:
is_valid = False
if isinstance( data.datatype, self.formats ):
if data.datatype.matches_any( self.formats ):
is_valid = True
else:
target_ext, converted_dataset = data.find_conversion_destination( self.formats )
lib/galaxy/util/__init__.py
+54 -3
View File
@@ -137,8 +137,60 @@ def parse_xml(fname):
def xml_to_string( elem, pretty=False ):
"""Returns a string from an xml tree"""
if pretty:
return ElementTree.tostring( pretty_print_xml( elem ) )
return ElementTree.tostring( elem )
elem = pretty_print_xml( elem )
try:
return ElementTree.tostring( elem )
except TypeError, e:
#assume this is a comment
if hasattr( elem, 'text' ):
return "<!-- %s -->\n" % ( elem.text )
else:
raise e
def xml_element_compare( elem1, elem2 ):
if not isinstance( elem1, dict ):
elem1 = xml_element_to_dict( elem1 )
if not isinstance( elem2, dict ):
elem2 = xml_element_to_dict( elem2 )
return elem1 == elem2
def xml_element_list_compare( elem_list1, elem_list2 ):
return [ xml_element_to_dict( elem ) for elem in elem_list1 ] == [ xml_element_to_dict( elem ) for elem in elem_list2 ]
def xml_element_to_dict( elem ):
rval = {}
if elem.attrib:
rval[ elem.tag ] = {}
else:
rval[ elem.tag ] = None
sub_elems = list( elem )
if sub_elems:
sub_elem_dict = dict()
for sub_sub_elem_dict in map( xml_element_to_dict, sub_elems ):
for key, value in sub_sub_elem_dict.iteritems():
if key not in sub_elem_dict:
sub_elem_dict[ key ] = []
sub_elem_dict[ key ].append( value )
for key, value in sub_elem_dict.iteritems():
if len( value ) == 1:
rval[ elem.tag ][ k ] = value[0]
else:
rval[ elem.tag ][ k ] = value
if elem.attrib:
for key, value in elem.attrib.iteritems():
rval[ elem.tag ][ "@%s" % key ] = value
if elem.text:
text = elem.text.strip()
if text and sub_elems or elem.attrib:
rval[ elem.tag ][ '#text' ] = text
else:
rval[ elem.tag ] = text
return rval
def pretty_print_xml( elem, level=0 ):
pad = ' '
@@ -287,7 +339,6 @@ def sanitize_param(value):
elif isinstance( value, list ):
return map(sanitize_text, value)
else:
print value
raise Exception, 'Unknown parameter type (%s)' % ( type( value ) )
valid_filename_chars = set( string.ascii_letters + string.digits + '_.' )
lib/galaxy/util/template.py
+1 -1
View File
@@ -6,4 +6,4 @@ from Cheetah.Template import Template
def fill_template( template_text, context=None, **kwargs ):
if not context:
context = kwargs
return str( Template( source=template_text, searchList=[context] ) )
return str( Template( source=template_text, searchList=[context] ) )
lib/galaxy/visualization/data_providers/basic.py
+1 -2
View File
@@ -178,10 +178,9 @@ class ColumnDataProvider( BaseDataProvider ):
meta[ 'mean' ] = float( meta[ 'sum' ] ) / count
sorted_data = sorted( response[ 'data' ][ index ] )
# even data count -
middle_index = ( count / 2 ) - 1
if count % 2 == 0:
meta[ 'median' ] = sum( sorted_data[ middle_index : ( middle_index + 1 ) ] ) / 2.0
meta[ 'median' ] = ( ( sorted_data[ middle_index ] + sorted_data[( middle_index + 1 )] ) / 2.0 )
else:
meta[ 'median' ] = sorted_data[ middle_index ]
lib/galaxy/visualization/data_providers/registry.py
+1 -1
View File
@@ -32,7 +32,7 @@ class DataProviderRegistry( object ):
"bigwig": genome.BigWigDataProvider,
"bigbed": genome.BigBedDataProvider,
"column": ColumnDataProvider
"column_with_stats": ColumnDataProvider
}
def get_data_provider( self, trans, name=None, source='data', raw=False, original_dataset=None ):
lib/galaxy/visualization/registry.py
+114 -76
View File
@@ -6,6 +6,7 @@ Lower level of visualization framework which does three main things:
"""
import os
import shutil
import glob
from galaxy import util
import galaxy.model
@@ -15,22 +16,28 @@ import logging
log = logging.getLogger( __name__ )
__TODO__ = """
BUGS:
anon users clicking a viz link gets 'must be' msg in galaxy_main (w/ masthead)
should not show visualizations (no icon)?
newick files aren't being sniffed prop? - datatype is txt
BUGS:
anon users clicking a viz link gets 'must be' msg in galaxy_main (w/ masthead)
should not show visualizations (no icon)?
newick files aren't being sniffed prop? - datatype is txt
have parsers create objects instead of dicts
allow data_sources with no model_class but have tests (isAdmin, etc.)
maybe that's an instance of User model_class?
some confused vocabulary in docs, var names
tests:
anding, grouping, not
data_sources:
lists of
add description element to visualization.
have parsers create objects instead of dicts
allow data_sources with no model_class but have tests (isAdmin, etc.)
maybe that's an instance of User model_class?
some confused vocabulary in docs, var names
tests:
anding, grouping, not
has_dataprovider
data_sources:
lists of
add description element to visualization.
TESTS to add:
has dataprovider
user is admin
"""
# ------------------------------------------------------------------- the registry
class VisualizationsRegistry( object ):
"""
Main responsibilities are:
@@ -57,10 +64,10 @@ class VisualizationsRegistry( object ):
return 'VisualizationsRegistry(%s)' %( listings_keys_str )
def __init__( self, galaxy_root, configuration_filepath ):
# load the registry from the given xml file using the given parser
# load the registry from the xml files located in configuration_filepath using the given parser
configuration_filepath = os.path.join( galaxy_root, configuration_filepath )
configuration_filepath = self.check_conf_filepath( configuration_filepath )
self.configuration_filepath = configuration_filepath
self.configuration_filepath = self.check_conf_filepath( configuration_filepath )
self.move_sample_conf_files()
self.load()
# what to use to parse query strings into resources/vars for the template
@@ -68,20 +75,27 @@ class VisualizationsRegistry( object ):
def check_conf_filepath( self, configuration_filepath ):
"""
If given file at filepath exists, return that filepath. If not,
see if filepath + '.sample' exists and, if so, copy that into filepath.
If neither original or sample exist, throw an IOError (currently,
this is a requireed file).
Checks for the existence of the given filepath.
:param configurarion_filepath: full filepath to the visualization config directory
:raises IOError: if the given directory doesn't exist
"""
if os.path.exists( configuration_filepath ):
return configuration_filepath
else:
sample_file = configuration_filepath + '.sample'
if os.path.exists( sample_file ):
shutil.copy2( sample_file, configuration_filepath )
return configuration_filepath
raise IOError( 'visualization configuration file (%s) not found' %( configuration_filepath ) )
if not os.path.exists( configuration_filepath ):
raise IOError( 'visualization configuration directory (%s) not found' %( configuration_filepath ) )
return configuration_filepath
def move_sample_conf_files( self ):
"""
Copies any `*.xml.sample` files in `configuration_filepath` to
`.xml` files of the same names if no file with that name already exists.
:returns: a list of the files moved
"""
files_moved = []
for sample_file in glob.glob( os.path.join( self.configuration_filepath, '*.sample' ) ):
new_name = os.path.splitext( sample_file )[0]
if not os.path.exists( new_name ):
shutil.copy2( sample_file, new_name )
files_moved.append( new_name )
def load( self ):
"""
@@ -93,6 +107,45 @@ class VisualizationsRegistry( object ):
"""
self.listings = VisualizationsConfigParser.parse( self.configuration_filepath )
def get_visualization( self, trans, visualization_name, target_object ):
"""
Return data to build a url to the visualization with the given
`visualization_name` if it's applicable to `target_object` or
`None` if it's not.
"""
# a little weird to pass trans because this registry is part of the trans.app
listing_data = self.listings.get( visualization_name, None )
if not listing_data:
return None
data_sources = listing_data[ 'data_sources' ]
for data_source in data_sources:
# currently a model class is required
model_class = data_source[ 'model_class' ]
if not isinstance( target_object, model_class ):
continue
# tests are optional - default is the above class test
tests = data_source[ 'tests' ]
if tests and not self.is_object_applicable( trans, target_object, tests ):
continue
param_data = data_source[ 'to_params' ]
url = self.get_visualization_url( trans, target_object, visualization_name, param_data )
link_text = listing_data.get( 'link_text', None )
if not link_text:
# default to visualization name, titlecase, and replace underscores
link_text = visualization_name.title().replace( '_', ' ' )
render_location = listing_data.get( 'render_location' )
# remap some of these vars for direct use in ui.js, PopupMenu (e.g. text->html)
return {
'href' : url,
'html' : link_text,
'target': render_location
}
return None
# -- building links to visualizations from objects --
def get_visualizations( self, trans, target_object ):
"""
@@ -100,36 +153,11 @@ class VisualizationsRegistry( object ):
the urls to call in order to render the visualizations.
"""
#TODO:?? a list of objects? YAGNI?
# a little weird to pass trans because this registry is part of the trans.app
applicable_visualizations = []
for vis_name, listing_data in self.listings.items():
data_sources = listing_data[ 'data_sources' ]
for data_source in data_sources:
# currently a model class is required
model_class = data_source[ 'model_class' ]
if not isinstance( target_object, model_class ):
continue
# tests are optional - default is the above class test
tests = data_source[ 'tests' ]
if tests and not self.is_object_applicable( trans, target_object, tests ):
continue
param_data = data_source[ 'to_params' ]
url = self.get_visualization_url( trans, target_object, vis_name, param_data )
link_text = listing_data.get( 'link_text', None )
if not link_text:
# default to visualization name, titlecase, and replace underscores
link_text = vis_name.title().replace( '_', ' ' )
render_location = listing_data.get( 'render_location' )
# remap some of these vars for direct use in ui.js, PopupMenu (e.g. text->html)
applicable_visualizations.append({
'href' : url,
'html' : link_text,
'target': render_location
})
for vis_name in self.listings:
url_data = self.get_visualization( trans, vis_name, target_object )
if url_data:
applicable_visualizations.append( url_data )
return applicable_visualizations
def is_object_applicable( self, trans, target_object, data_source_tests ):
@@ -151,10 +179,11 @@ class VisualizationsRegistry( object ):
# convert datatypes to their actual classes (for use with isinstance)
test_result = trans.app.datatypes_registry.get_datatype_class_by_name( test_result )
if not test_result:
# warn if can't find class, but continue
# warn if can't find class, but continue (with other tests)
log.warn( 'visualizations_registry cannot find class (%s) for applicability test', test_result )
continue
#NOTE: tests are OR'd, if any test passes - the visualization can be applied
if test_fn( target_object, test_result ):
#log.debug( 'test passed' )
return True
@@ -254,11 +283,11 @@ class VisualizationsConfigParser( object ):
VALID_RENDER_LOCATIONS = [ 'galaxy_main', '_top', '_blank' ]
@classmethod
def parse( cls, xml_filepath, debug=True ):
def parse( cls, config_dir, debug=True ):
"""
Static class interface
Static class interface.
"""
return cls( debug ).parse_file( xml_filepath )
return cls( debug ).parse_files( config_dir )
def __init__( self, debug=False ):
self.debug = debug
@@ -268,38 +297,47 @@ class VisualizationsConfigParser( object ):
self.param_parser = ParamParser()
self.param_modifier_parser = ParamModifierParser()
def parse_file( self, xml_filepath ):
def parse_files( self, config_dir ):
"""
Parse the given XML file for visualizations data.
Parse each XML file in `config_dir` for visualizations config data.
If an error occurs while parsing a visualizations entry it is skipped.
If an error occurs while parsing a visualizations entry, it is skipped.
:returns: registry data in dictionary form
"""
returned = {}
try:
xml_tree = galaxy.util.parse_xml( xml_filepath )
for visualization_conf in xml_tree.getroot().findall( 'visualization' ):
visualization = None
visualization_name = visualization_conf.get( 'name' )
for xml_filepath in glob.glob( os.path.join( config_dir, '*.xml' ) ):
try:
visualization = self.parse_visualization( visualization_conf )
visualization_name, visualization = self.parse_file( xml_filepath )
# skip vis' with parsing errors - don't shutdown the startup
except ParsingException, parse_exc:
log.error( 'Skipped visualization configuration "%s" due to parsing errors: %s',
visualization_name, str( parse_exc ), exc_info=self.debug )
log.error( 'Skipped visualization config "%s" due to parsing errors: %s',
xml_filepath, str( parse_exc ), exc_info=self.debug )
if visualization:
returned[ visualization_name ] = visualization
log.debug( 'Visualization config loaded for: %s', visualization_name )
except Exception, exc:
log.error( 'Error parsing visualization configuration file %s: %s',
xml_filepath, str( exc ), exc_info=( not self.debug ) )
#TODO: change when this is required
log.error( 'Error parsing visualizations configuration directory %s: %s',
config_dir, str( exc ), exc_info=( not self.debug ) )
#TODO: change when this framework is on by default
if self.debug:
raise
return returned
def parse_file( self, xml_filepath ):
"""
Parse the given XML file for visualizations data.
:returns: tuple of ( `visualization_name`, `visualization` )
"""
xml_tree = galaxy.util.parse_xml( xml_filepath )
visualization_conf = xml_tree.getroot()
visualization_name = visualization_conf.get( 'name' )
visualization = self.parse_visualization( visualization_conf )
return visualization_name, visualization
def parse_visualization( self, xml_tree ):
"""
Parse the template, name, and any data_sources and params from the
lib/galaxy/web/base/controller.py
+161 -97
View File
@@ -5,32 +5,43 @@ import logging
import operator
import os
import re
import pkg_resources
import urllib
from gettext import gettext
import pkg_resources
pkg_resources.require("SQLAlchemy >= 0.4")
from sqlalchemy import func, and_, select
pkg_resources.require( "Routes" )
import routes
from sqlalchemy import func, and_, select
from paste.httpexceptions import HTTPBadRequest, HTTPInternalServerError, HTTPNotImplemented, HTTPRequestRangeNotSatisfiable
from paste.httpexceptions import HTTPBadRequest, HTTPInternalServerError
from paste.httpexceptions import HTTPNotImplemented, HTTPRequestRangeNotSatisfiable
from galaxy.exceptions import ItemAccessibilityException, ItemDeletionException, ItemOwnershipException
from galaxy.exceptions import MessageException
from galaxy import web
from galaxy import model
from galaxy import security
from galaxy import util
from galaxy import util, web, model
from gettext import gettext
from galaxy.datatypes.interval import ChromatinInteractions
from galaxy.exceptions import ItemAccessibilityException, ItemDeletionException, ItemOwnershipException, MessageException
from galaxy.security.validate_user_input import validate_publicname
from galaxy.util.sanitize_html import sanitize_html
from galaxy.visualization.genome.visual_analytics import get_tool_def
from galaxy.web import error, url_for
from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField
from galaxy.web.form_builder import build_select_field, HistoryField, PasswordField, WorkflowField, WorkflowMappingField
from galaxy.workflow.modules import module_factory
from galaxy.model.orm import eagerload, eagerload_all
from galaxy.security.validate_user_input import validate_publicname
from galaxy.util.sanitize_html import sanitize_html
from galaxy.datatypes.interval import ChromatinInteractions
from galaxy.datatypes.data import Text
from galaxy.visualization.genome.visual_analytics import get_tool_def
from galaxy.datatypes.display_applications import util as da_util
from galaxy.datatypes.metadata import FileParameter
log = logging.getLogger( __name__ )
# States for passing messages
@@ -102,23 +113,34 @@ class BaseController( object ):
try:
decoded_id = trans.security.decode_id( id )
except:
raise MessageException( "Malformed %s id ( %s ) specified, unable to decode" % ( class_name, str( id ) ), type='error' )
raise MessageException( "Malformed %s id ( %s ) specified, unable to decode"
% ( class_name, str( id ) ), type='error' )
try:
item_class = self.get_class( class_name )
assert item_class is not None
item = trans.sa_session.query( item_class ).get( decoded_id )
assert item is not None
except:
log.exception( "Invalid %s id ( %s ) specified" % ( class_name, id ) )
except Exception, exc:
log.exception( "Invalid %s id ( %s ) specified: %s" % ( class_name, id, str( exc ) ) )
raise MessageException( "Invalid %s id ( %s ) specified" % ( class_name, id ), type="error" )
if check_ownership or check_accessible:
self.security_check( trans, item, check_ownership, check_accessible, id )
self.security_check( trans, item, check_ownership, check_accessible )
if deleted == True and not item.deleted:
raise ItemDeletionException( '%s "%s" is not deleted' % ( class_name, getattr( item, 'name', id ) ), type="warning" )
raise ItemDeletionException( '%s "%s" is not deleted'
% ( class_name, getattr( item, 'name', id ) ), type="warning" )
elif deleted == False and item.deleted:
raise ItemDeletionException( '%s "%s" is deleted' % ( class_name, getattr( item, 'name', id ) ), type="warning" )
raise ItemDeletionException( '%s "%s" is deleted'
% ( class_name, getattr( item, 'name', id ) ), type="warning" )
return item
# this should be here - but catching errors from sharable item controllers that *should* have SharableItemMixin
# but *don't* then becomes difficult
#def security_check( self, trans, item, check_ownership=False, check_accessible=False ):
# log.warn( 'BaseController.security_check: %s, %b, %b', str( item ), check_ownership, check_accessible )
# # meant to be overridden in SharableSecurityMixin
# return item
def get_user( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ):
return self.get_object( trans, id, 'User', check_ownership=False, check_accessible=False, deleted=deleted )
@@ -128,7 +150,7 @@ class BaseController( object ):
def get_role( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ):
return self.get_object( trans, id, 'Role', check_ownership=False, check_accessible=False, deleted=deleted )
def encode_all_ids( self, trans, rval ):
def encode_all_ids( self, trans, rval, recursive=False ):
"""
Encodes all integer values in the dict rval whose keys are 'id' or end with '_id'
@@ -142,6 +164,9 @@ class BaseController( object ):
rval[k] = trans.security.encode_id( v )
except:
pass # probably already encoded
else:
if recursive and type(v) == dict:
rval[k] = self.encode_all_ids(trans, v, recursive)
return rval
Root = BaseController
@@ -151,7 +176,9 @@ class BaseUIController( BaseController ):
def get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ):
try:
return BaseController.get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None )
return BaseController.get_object( self, trans, id, class_name,
check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted )
except MessageException:
raise # handled in the caller
except:
@@ -163,13 +190,15 @@ class BaseAPIController( BaseController ):
def get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ):
try:
return BaseController.get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None )
return BaseController.get_object( self, trans, id, class_name,
check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted )
except ItemDeletionException, e:
raise HTTPBadRequest( detail="Invalid %s id ( %s ) specified" % ( class_name, str( id ) ) )
raise HTTPBadRequest( detail="Invalid %s id ( %s ) specified: %s" % ( class_name, str( id ), str( e ) ) )
except MessageException, e:
raise HTTPBadRequest( detail=e.err_msg )
except Exception, e:
log.exception( "Execption in get_object check for %s %s:" % ( class_name, str( id ) ) )
log.exception( "Execption in get_object check for %s %s: %s" % ( class_name, str( id ), str( e ) ) )
raise HTTPInternalServerError( comment=str( e ) )
def validate_in_users_and_groups( self, trans, payload ):
@@ -221,24 +250,30 @@ class Datatype( object ):
# -- Mixins for working with Galaxy objects. --
#
class SharableItemSecurityMixin:
""" Mixin for handling security for sharable items. """
def security_check( self, trans, item, check_ownership=False, check_accessible=False ):
""" Security checks for an item: checks if (a) user owns item or (b) item is accessible to user. """
# all items are accessible to an admin
if trans.user and trans.user_is_admin():
return item
# Verify ownership: there is a current user and that user is the same as the item's
if check_ownership:
# Verify ownership.
if not trans.user:
raise ItemOwnershipException( "Must be logged in to manage Galaxy items", type='error' )
if item.user != trans.user:
raise ItemOwnershipException( "%s is not owned by the current user" % item.__class__.__name__, type='error' )
# Verify accessible:
# if it's part of a lib - can they access via security
# if it's something else (sharable) have they been added to the item's users_shared_with_dot_users
if check_accessible:
if type( item ) in ( trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset ):
if not ( trans.user_is_admin() or trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), item, trans.user ) ):
if not trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), item, trans.user ):
raise ItemAccessibilityException( "%s is not accessible to the current user" % item.__class__.__name__, type='error' )
else:
# Verify accessible.
if ( item.user != trans.user ) and ( not item.importable ) and ( trans.user not in item.users_shared_with_dot_users ):
raise ItemAccessibilityException( "%s is not accessible to the current user" % item.__class__.__name__, type='error' )
return item
@@ -249,8 +284,10 @@ class UsesHistoryMixin( SharableItemSecurityMixin ):
def get_history( self, trans, id, check_ownership=True, check_accessible=False, deleted=None ):
"""Get a History from the database by id, verifying ownership."""
history = self.get_object( trans, id, 'History', check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted )
return self.security_check( trans, history, check_ownership, check_accessible )
history = self.get_object( trans, id, 'History',
check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted )
history = self.security_check( trans, history, check_ownership, check_accessible )
return history
def get_history_datasets( self, trans, history, show_deleted=False, show_hidden=False, show_purged=False ):
""" Returns history's datasets. """
@@ -475,7 +512,8 @@ class UsesHistoryDatasetAssociationMixin:
"""
#TODO: duplicate of above? alias to above (or vis-versa)
self.security_check( trans, history, check_ownership=check_ownership, check_accessible=check_accessible )
hda = self.get_object( trans, dataset_id, 'HistoryDatasetAssociation', check_ownership=False, check_accessible=False, deleted=False )
hda = self.get_object( trans, dataset_id, 'HistoryDatasetAssociation',
check_ownership=False, check_accessible=False )
if check_accessible:
if not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset ):
@@ -593,72 +631,6 @@ class UsesHistoryDatasetAssociationMixin:
return trans.security.encode_dict_ids( hda_dict )
def profile_get_hda_dict( self, trans, hda ):
"""Profiles returning full details of this HDA in dictionary form.
"""
from galaxy.util.debugging import SimpleProfiler
profiler = SimpleProfiler()
profiler.start()
hda_dict = hda.get_api_value( view='element' )
profiler.report( '\t\t get_api_value' )
history = hda.history
hda_dict[ 'api_type' ] = "file"
# Add additional attributes that depend on trans can hence must be added here rather than at the model level.
can_access_hda = trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset )
can_access_hda = ( trans.user_is_admin() or can_access_hda )
hda_dict[ 'accessible' ] = can_access_hda
profiler.report( '\t\t accessible' )
# ---- return here if deleted AND purged OR can't access
purged = ( hda.purged or hda.dataset.purged )
if ( hda.deleted and purged ) or not can_access_hda:
#TODO: get_api_value should really go AFTER this - only summary data
return ( profiler, trans.security.encode_dict_ids( hda_dict ) )
if trans.user_is_admin() or trans.app.config.expose_dataset_path:
hda_dict[ 'file_name' ] = hda.file_name
profiler.report( '\t\t file_name' )
hda_dict[ 'download_url' ] = url_for( 'history_contents_display',
history_id = trans.security.encode_id( history.id ),
history_content_id = trans.security.encode_id( hda.id ) )
profiler.report( '\t\t download_url' )
# indeces, assoc. metadata files, etc.
meta_files = []
for meta_type in hda.metadata.spec.keys():
if isinstance( hda.metadata.spec[ meta_type ].param, FileParameter ):
meta_files.append( dict( file_type=meta_type ) )
if meta_files:
hda_dict[ 'meta_files' ] = meta_files
profiler.report( '\t\t meta_files' )
# currently, the viz reg is optional - handle on/off
if trans.app.visualizations_registry:
hda_dict[ 'visualizations' ] = trans.app.visualizations_registry.get_visualizations( trans, hda )
else:
hda_dict[ 'visualizations' ] = hda.get_visualizations()
profiler.report( '\t\t visualizations' )
#TODO: it may also be wiser to remove from here and add as API call that loads the visualizations
# when the visualizations button is clicked (instead of preloading/pre-checking)
# ---- return here if deleted
if hda.deleted and not purged:
return ( profiler, trans.security.encode_dict_ids( hda_dict ) )
# if a tool declares 'force_history_refresh' in its xml, when the hda -> ready, reload the history panel
# expensive
if( ( hda.state in [ 'running', 'queued' ] )
and ( hda.creating_job and hda.creating_job.tool_id ) ):
tool_used = trans.app.toolbox.get_tool( hda.creating_job.tool_id )
if tool_used and tool_used.force_history_refresh:
hda_dict[ 'force_history_refresh' ] = True
profiler.report( '\t\t force_history_refresh' )
return ( profiler, trans.security.encode_dict_ids( hda_dict ) )
def get_hda_dict_with_error( self, trans, hda, error_msg='' ):
return trans.security.encode_dict_ids({
'id' : hda.id,
@@ -742,13 +714,105 @@ class UsesLibraryMixin:
class UsesLibraryMixinItems( SharableItemSecurityMixin ):
def get_library_folder( self, trans, id, check_ownership=False, check_accessible=True ):
return self.get_object( trans, id, 'LibraryFolder', check_ownership=False, check_accessible=check_accessible )
return self.get_object( trans, id, 'LibraryFolder',
check_ownership=False, check_accessible=check_accessible )
def get_library_dataset_dataset_association( self, trans, id, check_ownership=False, check_accessible=True ):
return self.get_object( trans, id, 'LibraryDatasetDatasetAssociation', check_ownership=False, check_accessible=check_accessible )
return self.get_object( trans, id, 'LibraryDatasetDatasetAssociation',
check_ownership=False, check_accessible=check_accessible )
def get_library_dataset( self, trans, id, check_ownership=False, check_accessible=True ):
return self.get_object( trans, id, 'LibraryDataset', check_ownership=False, check_accessible=check_accessible )
return self.get_object( trans, id, 'LibraryDataset',
check_ownership=False, check_accessible=check_accessible )
#TODO: it makes no sense that I can get roles from a user but not user.is_admin()
#def can_user_add_to_library_item( self, trans, user, item ):
# if not user: return False
# return ( ( user.is_admin() )
# or ( trans.app.security_agent.can_add_library_item( user.all_roles(), item ) ) )
def can_current_user_add_to_library_item( self, trans, item ):
if not trans.user: return False
return ( ( trans.user_is_admin() )
or ( trans.app.security_agent.can_add_library_item( trans.get_current_user_roles(), item ) ) )
def copy_hda_to_library_folder( self, trans, hda, library_folder, roles=None, ldda_message='' ):
#PRECONDITION: permissions for this action on hda and library_folder have been checked
roles = roles or []
# this code was extracted from library_common.add_history_datasets_to_library
#TODO: refactor library_common.add_history_datasets_to_library to use this for each hda to copy
# create the new ldda and apply the folder perms to it
ldda = hda.to_library_dataset_dataset_association( trans, target_folder=library_folder,
roles=roles, ldda_message=ldda_message )
self._apply_library_folder_permissions_to_ldda( trans, library_folder, ldda )
self._apply_hda_permissions_to_ldda( trans, hda, ldda )
#TODO:?? not really clear on how permissions are being traded here
# seems like hda -> ldda permissions should be set in to_library_dataset_dataset_association
# then they get reset in _apply_library_folder_permissions_to_ldda
# then finally, re-applies hda -> ldda for missing actions in _apply_hda_permissions_to_ldda??
return ldda
def _apply_library_folder_permissions_to_ldda( self, trans, library_folder, ldda ):
"""
Copy actions/roles from library folder to an ldda (and it's library_dataset).
"""
#PRECONDITION: permissions for this action on library_folder and ldda have been checked
security_agent = trans.app.security_agent
security_agent.copy_library_permissions( trans, library_folder, ldda )
security_agent.copy_library_permissions( trans, library_folder, ldda.library_dataset )
return security_agent.get_permissions( ldda )
def _apply_hda_permissions_to_ldda( self, trans, hda, ldda ):
"""
Copy actions/roles from hda to ldda.library_dataset (and then ldda) if ldda
doesn't already have roles for the given action.
"""
#PRECONDITION: permissions for this action on hda and ldda have been checked
# Make sure to apply any defined dataset permissions, allowing the permissions inherited from the
# library_dataset to over-ride the same permissions on the dataset, if they exist.
security_agent = trans.app.security_agent
dataset_permissions_dict = security_agent.get_permissions( hda.dataset )
library_dataset = ldda.library_dataset
library_dataset_actions = [ permission.action for permission in library_dataset.actions ]
# except that: if DATASET_MANAGE_PERMISSIONS exists in the hda.dataset permissions,
# we need to instead apply those roles to the LIBRARY_MANAGE permission to the library dataset
dataset_manage_permissions_action = security_agent.get_action( 'DATASET_MANAGE_PERMISSIONS' ).action
library_manage_permissions_action = security_agent.get_action( 'LIBRARY_MANAGE' ).action
#TODO: test this and remove if in loop below
#TODO: doesn't handle action.action
#if dataset_manage_permissions_action in dataset_permissions_dict:
# managing_roles = dataset_permissions_dict.pop( dataset_manage_permissions_action )
# dataset_permissions_dict[ library_manage_permissions_action ] = managing_roles
flush_needed = False
for action, dataset_permissions_roles in dataset_permissions_dict.items():
if isinstance( action, security.Action ):
action = action.action
# alter : DATASET_MANAGE_PERMISSIONS -> LIBRARY_MANAGE (see above)
if action == dataset_manage_permissions_action:
action = library_manage_permissions_action
#TODO: generalize to util.update_dict_without_overwrite
# add the hda actions & roles to the library_dataset
#NOTE: only apply an hda perm if it's NOT set in the library_dataset perms (don't overwrite)
if action not in library_dataset_actions:
for role in dataset_permissions_roles:
ldps = trans.model.LibraryDatasetPermissions( action, library_dataset, role )
ldps = [ ldps ] if not isinstance( ldps, list ) else ldps
for ldp in ldps:
trans.sa_session.add( ldp )
flush_needed = True
if flush_needed:
trans.sa_session.flush()
# finally, apply the new library_dataset to it's associated ldda (must be the same)
security_agent.copy_library_permissions( trans, library_dataset, ldda )
return security_agent.get_permissions( ldda )
class UsesVisualizationMixin( UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems ):
lib/galaxy/web/framework/__init__.py
+3 -3
View File
@@ -367,9 +367,9 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ):
client_locales = self.environ['HTTP_ACCEPT_LANGUAGE'].split( ',' )
for locale in client_locales:
try:
locales.append( Locale.parse( locale.split( ';' )[0], sep='-' ).language )
except UnknownLocaleError:
pass
locales.append( Locale.parse( locale.split( ';' )[0].strip(), sep='-' ).language )
except Exception, e:
log.debug( "Error parsing locale '%s'. %s: %s", locale, type( e ), e )
if not locales:
# Default to English
locales = 'en'
lib/galaxy/web/framework/helpers/grids.py
+11 -2
View File
@@ -225,7 +225,13 @@ class Grid( object ):
params = cur_filter_dict.copy()
params['sort'] = sort_key
params['async'] = ( 'async' in kwargs )
trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params )
#TODO:??
# commenting this out; when this fn calls session.add( action ) and session.flush the query from this fn
# is effectively 'wiped' out. Nate believes it has something to do with our use of session( autocommit=True )
# in mapping.py. If you change that to False, the log_action doesn't affect the query
# Below, I'm rendering the template first (that uses query), then calling log_action, then returning the page
#trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params )
# Render grid.
def url( *args, **kwargs ):
@@ -260,7 +266,7 @@ class Grid( object ):
# utf-8 unicode; however, this would require encoding the object as utf-8 before returning the grid
# results via a controller method, which is require substantial changes. Hence, for now, return grid
# as str.
return trans.fill_template( iff( async_request, self.async_template, self.template ),
page = trans.fill_template( iff( async_request, self.async_template, self.template ),
grid=self,
query=query,
cur_page_num = page_num,
@@ -280,6 +286,9 @@ class Grid( object ):
# Pass back kwargs so that grid template can set and use args without
# grid explicitly having to pass them.
kwargs=kwargs )
trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params )
return page
def get_ids( self, **kwargs ):
id = []
if 'id' in kwargs:
lib/galaxy/webapps/galaxy/api/datasets.py
+25 -6
View File
@@ -6,6 +6,7 @@ from galaxy.visualization.data_providers.genome import FeatureLocationIndexDataP
from galaxy.web.base.controller import BaseAPIController, UsesVisualizationMixin, UsesHistoryDatasetAssociationMixin
from galaxy.web.base.controller import UsesHistoryMixin
from galaxy.web.framework.helpers import is_true
from galaxy.datatypes import dataproviders
import logging
log = logging.getLogger( __name__ )
@@ -169,12 +170,16 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory
return { 'dataset_type': indexer.dataset_type, 'data': None }
else:
return stats
# Stats provides features/base and resolution is bases/pixel, so
# multiplying them yields features/pixel.
features_per_pixel = stats[ 'data' ][ 'max' ] * float( kwargs[ 'resolution' ] )
# Use heuristic based on max depth and region size to determine whether to
# return coverage data. When zoomed out and region is large, max depth
# Use heuristic based on features/pixel and region size to determine whether to
# return coverage data. When zoomed out and region is large, features/pixel
# is determining factor. However, when sufficiently zoomed in and region is
# small, coverage data is no longer provided.
if int( high ) - int( low ) > 50000 and stats[ 'data' ][ 'max' ] > 1000:
if int( high ) - int( low ) > 50000 and features_per_pixel > 1000:
return indexer.get_data( chrom, low, high )
#
@@ -213,10 +218,24 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory
return msg
registry = trans.app.data_provider_registry
# allow the caller to specifiy which provider is used
if provider and provider in registry.dataset_type_name_to_data_provider:
data_provider = registry.dataset_type_name_to_data_provider[ provider ]( dataset )
# or have it look up by datatype
# pulling from the original providers if possible, then the new providers
if provider:
if provider in registry.dataset_type_name_to_data_provider:
data_provider = registry.dataset_type_name_to_data_provider[ provider ]( dataset )
elif dataset.datatype.has_dataprovider( provider ):
kwargs = dataset.datatype.dataproviders[ provider ].parse_query_string_settings( kwargs )
# use dictionary to allow more than the data itself to be returned (data totals, other meta, etc.)
return {
'data': list( dataset.datatype.dataprovider( dataset, provider, **kwargs ) )
}
else:
raise dataproviders.exceptions.NoProviderAvailable( dataset.datatype, provider )
# no provider name: look up by datatype
else:
data_provider = registry.get_data_provider( trans, raw=True, original_dataset=dataset )
lib/galaxy/webapps/galaxy/api/genomes.py
+14 -8
View File
@@ -49,19 +49,14 @@ class GenomesController( BaseAPIController ):
Download and/or index a genome.
Parameters::
dbkey DB key of the build to download, ignored unless 'UCSC' is specified as the source
ncbi_name NCBI's genome identifier, ignored unless NCBI is specified as the source
ensembl_dbkey Ensembl's genome identifier, ignored unless Ensembl is specified as the source
url_dbkey DB key to use for this build, ignored unless URL is specified as the source
source Data source for this build. Can be: UCSC, Ensembl, NCBI, URL
liftover None or array of liftover url partial paths
dbkey DB key of the build to download
indexers POST array of indexers to run after downloading (indexers[] = first, indexers[] = second, ...)
longname a more descriptive name for the genome
func Allowed values:
'download' Download and index
'index' Index only
Returns::
If no error:
dict( status: 'ok', job: <job ID> )
@@ -69,6 +64,17 @@ class GenomesController( BaseAPIController ):
dict( status: 'error', error: <error message> )
"""
#??: Planned?
#Parameters::
# dbkey DB key of the build to download, ignored unless 'UCSC' is specified as the source
# ncbi_name NCBI's genome identifier, ignored unless NCBI is specified as the source
# ensembl_dbkey Ensembl's genome identifier, ignored unless Ensembl is specified as the source
# url_dbkey DB key to use for this build, ignored unless URL is specified as the source
# source Data source for this build. Can be: UCSC, Ensembl, NCBI, URL
# indexers POST array of indexers to run after downloading (indexers[] = first, indexers[] = second, ...)
# func Allowed values:
# 'download' Download and index
# 'index' Index only
params = util.Params( payload )
from galaxy.web.controllers.data_admin import build_param_dict as massage
paramdict = massage( params, trans )
lib/galaxy/webapps/galaxy/api/histories.py
+53 -25
View File
@@ -2,6 +2,10 @@
API operations on a history.
"""
import pkg_resources
pkg_resources.require( "Paste" )
from paste.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPInternalServerError, HTTPException
from galaxy import web
from galaxy.util import string_as_bool, restore_text
from galaxy.util.sanitize_html import sanitize_html
@@ -75,9 +79,13 @@ class HistoriesController( BaseAPIController, UsesHistoryMixin ):
history_data = self.get_history_dict( trans, history )
history_data[ 'contents_url' ] = url_for( 'history_contents', history_id=history_id )
except HTTPBadRequest, bad_req:
trans.response.status = 400
return str( bad_req )
except Exception, e:
msg = "Error in history API at showing history detail: %s" % ( str( e ) )
log.error( msg, exc_info=True )
log.exception( msg, exc_info=True )
trans.response.status = 500
return msg
@@ -116,33 +124,53 @@ class HistoriesController( BaseAPIController, UsesHistoryMixin ):
if kwd.get( 'payload', None ):
purge = string_as_bool( kwd['payload'].get( 'purge', False ) )
rval = { 'id' : history_id }
try:
history = self.get_history( trans, history_id, check_ownership=True, check_accessible=False, deleted=True )
except Exception, e:
return str( e )
history = self.get_history( trans, history_id, check_ownership=True, check_accessible=False )
history.deleted = True
history.deleted = True
if purge and trans.app.config.allow_user_dataset_purge:
# First purge all the datasets
for hda in history.datasets:
if hda.purged:
continue
hda.purged = True
trans.sa_session.add( hda )
trans.sa_session.flush()
if hda.dataset.user_can_purge:
try:
hda.dataset.full_delete()
trans.sa_session.add( hda.dataset )
except:
pass
if purge:
if not trans.app.config.allow_user_dataset_purge:
raise HTTPForbidden( detail='This instance does not allow user dataset purging' )
# First purge all the datasets
for hda in history.datasets:
if hda.purged:
continue
hda.purged = True
trans.sa_session.add( hda )
trans.sa_session.flush()
# Now mark the history as purged
history.purged = True
self.sa_session.add( history )
trans.sa_session.flush()
return 'OK'
if hda.dataset.user_can_purge:
try:
hda.dataset.full_delete()
trans.sa_session.add( hda.dataset )
except:
pass
# flush now to preserve deleted state in case of later interruption
trans.sa_session.flush()
# Now mark the history as purged
history.purged = True
self.sa_session.add( history )
rval[ 'purged' ] = True
trans.sa_session.flush()
rval[ 'deleted' ] = True
except HTTPInternalServerError, http_server_err:
log.exception( 'Histories API, delete: uncaught HTTPInternalServerError: %s, %s\n%s',
history_id, str( kwd ), str( http_server_err ) )
raise
except HTTPException, http_exc:
raise
except Exception, exc:
log.exception( 'Histories API, delete: uncaught exception: %s, %s\n%s',
history_id, str( kwd ), str( exc ) )
trans.response.status = 500
rval.update({ 'error': str( exc ) })
return rval
@web.expose_api
def undelete( self, trans, id, **kwd ):
@@ -165,7 +193,7 @@ class HistoriesController( BaseAPIController, UsesHistoryMixin ):
"""
#TODO: PUT /api/histories/{encoded_history_id} payload = { rating: rating } (w/ no security checks)
try:
history = self.get_history( trans, id, check_ownership=True, check_accessible=True, deleted=True )
history = self.get_history( trans, id, check_ownership=True, check_accessible=True )
# validation handled here and some parsing, processing, and conversion
payload = self._validate_and_parse_update_payload( payload )
# additional checks here (security, etc.)
lib/galaxy/webapps/galaxy/api/library_contents.py
+58 -3
View File
@@ -2,13 +2,18 @@
API operations on the contents of a library.
"""
import logging
from galaxy import web
from galaxy.model import ExtendedMetadata, ExtendedMetadataIndex
from galaxy.web.base.controller import BaseAPIController, HTTPBadRequest, url_for, UsesLibraryMixin, UsesLibraryMixinItems
from galaxy.web.base.controller import BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems
from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin
from galaxy.web.base.controller import HTTPBadRequest, url_for
from galaxy import util
log = logging.getLogger( __name__ )
class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems ):
class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems,
UsesHistoryDatasetAssociationMixin ):
@web.expose_api
# TODO: Add parameter to only get top level of datasets/subfolders.
@@ -32,7 +37,8 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar
rval.extend( traverse( subfolder ) )
for ld in folder.datasets:
if not admin:
can_access = trans.app.security_agent.can_access_dataset( current_user_roles, ld.library_dataset_dataset_association.dataset )
can_access = trans.app.security_agent.can_access_dataset(
current_user_roles, ld.library_dataset_dataset_association.dataset )
if (admin or can_access) and not ld.deleted:
log.debug( "type(folder): %s" % type( folder ) )
log.debug( "type(api_path): %s; folder.api_path: %s" % ( type(folder.api_path), folder.api_path ) )
@@ -99,6 +105,7 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar
if create_type not in ( 'file', 'folder' ):
trans.response.status = 400
return "Invalid value for 'create_type' parameter ( %s ) specified." % create_type
if 'folder_id' not in payload:
trans.response.status = 400
return "Missing requred 'folder_id' parameter."
@@ -113,6 +120,12 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar
# The rest of the security happens in the library_common controller.
real_folder_id = trans.security.encode_id( parent.id )
# are we copying an HDA to the library folder?
# we'll need the id and any message to attach, then branch to that private function
from_hda_id, ldda_message = ( payload.pop( 'from_hda_id', None ), payload.pop( 'ldda_message', '' ) )
if create_type == 'file' and from_hda_id:
return self._copy_hda_to_library_folder( trans, from_hda_id, library_id, real_folder_id, ldda_message )
#check for extended metadata, store it and pop it out of the param
#otherwise sanitize_param will have a fit
ex_meta_payload = None
@@ -180,6 +193,48 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar
#for cross type comparisions, ie "True" == True
yield prefix, ("%s" % (meta)).encode("utf8", errors='replace')
def _copy_hda_to_library_folder( self, trans, from_hda_id, library_id, folder_id, ldda_message='' ):
"""
Copies hda `from_hda_id` to library folder `library_folder_id` optionally
adding `ldda_message` to the new ldda's `message`.
`library_contents.create` will branch to this if called with 'from_hda_id'
in it's payload.
"""
log.debug( '_copy_hda_to_library_folder: %s' %( str(( from_hda_id, library_id, folder_id, ldda_message )) ) )
#PRECONDITION: folder_id has already been altered to remove the folder prefix ('F')
#TODO: allow name and other, editable ldda attrs?
if ldda_message:
ldda_message = util.sanitize_html.sanitize_html( ldda_message, 'utf-8' )
rval = {}
try:
# check permissions on (all three?) resources: hda, library, folder
#TODO: do we really need the library??
hda = self.get_dataset( trans, from_hda_id, check_ownership=True, check_accessible=True, check_state=True )
library = self.get_library( trans, library_id, check_accessible=True )
folder = self.get_library_folder( trans, folder_id, check_accessible=True )
if not self.can_current_user_add_to_library_item( trans, folder ):
trans.response.status = 403
return { 'error' : 'user has no permission to add to library folder (%s)' %( folder_id ) }
ldda = self.copy_hda_to_library_folder( trans, hda, folder, ldda_message=ldda_message )
ldda_dict = ldda.get_api_value()
rval = trans.security.encode_dict_ids( ldda_dict )
except Exception, exc:
#TODO: grrr...
if 'not accessible to the current user' in str( exc ):
trans.response.status = 403
return { 'error' : str( exc ) }
else:
log.exception( exc )
trans.response.status = 500
return { 'error' : str( exc ) }
return rval
@web.expose_api
def update( self, trans, id, library_id, payload, **kwd ):
"""
lib/galaxy/webapps/galaxy/api/provenance.py
+78
View File
@@ -0,0 +1,78 @@
"""
API operations provenance
"""
import logging
from galaxy import web
from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin
from paste.httpexceptions import HTTPNotImplemented, HTTPBadRequest
log = logging.getLogger( __name__ )
class BaseProvenanceController( BaseAPIController, UsesHistoryMixin ):
"""
"""
@web.expose_api
def index( self, trans, **kwd ):
follow = kwd.get('follow', False)
value = self._get_provenance( trans, self.provenance_item_class, kwd[self.provenance_item_id], follow )
return value
@web.expose_api
def show( self, trans, elem_name, **kwd ):
follow = kwd.get('follow', False)
value = self._get_provenance( trans, self.provenance_item_class, kwd[self.provenance_item_id], follow )
return value
@web.expose_api
def create( self, trans, tag_name, payload={}, **kwd ):
raise HTTPNotImplemented()
@web.expose_api
def delete( self, trans, tag_name, **kwd ):
raise HTTPBadRequest("Cannot Delete Provenance")
def _get_provenance(self, trans, item_class_name, item_id, follow=True):
provenance_item = self.get_object( trans, item_id, item_class_name, check_ownership=False, check_accessible=True )
out = self._get_record(trans, provenance_item, follow)
return out
def _get_record(self, trans, item, follow):
if item is not None:
if item.copied_from_library_dataset_dataset_association:
item = item.copied_from_library_dataset_dataset_association
return {
"id" : trans.security.encode_id(item.id),
"uuid" : ( lambda uuid: str( uuid ) if uuid else None )( item.dataset.uuid),
"tool_id" : item.creating_job.tool_id,
"parameters" : self._get_job_record(trans, item.creating_job, follow)
}
return None
def _get_job_record(self, trans, job, follow):
out = {}
for p in job.parameters:
out[p.name] = p.value
for in_d in job.input_datasets:
if follow:
out[in_d.name] = self._get_record(trans, in_d.dataset, follow)
else:
out[in_d.name] = {
"id" : trans.security.encode_id(in_d.dataset.id),
"uuid" : ( lambda uuid: str( uuid ) if uuid else None )( in_d.dataset.dataset.uuid )
}
return out
class HDAProvenanceController( BaseProvenanceController ):
controller_name = "history_content_provenance"
provenance_item_class = "HistoryDatasetAssociation"
provenance_item_id = "history_content_id"
class LDDAProvenanceController( BaseProvenanceController ):
controller_name = "ldda_provenance"
provenance_item_class = "LibraryDatasetDatasetAssociation"
provenance_item_id = "library_content_id"
lib/galaxy/webapps/galaxy/api/search.py
+5 -4
View File
@@ -30,7 +30,6 @@ class SearchController( BaseAPIController, SharableItemSecurityMixin ):
current_user_roles = trans.get_current_user_roles()
try:
results = query.process(trans)
print results
except Exception, e:
return {'error' : str(e)}
for item in results:
@@ -41,11 +40,13 @@ class SearchController( BaseAPIController, SharableItemSecurityMixin ):
if type( item ) in ( trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset ):
if (trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), item, trans.user ) ):
append = True
if not append:
if hasattr(item, 'dataset'):
elif type( item ) in trans.app.model.Job:
if item.used_id == trans.user or trans.user_is_admin():
append = True
elif hasattr(item, 'dataset'):
if trans.app.security_agent.can_access_dataset( current_user_roles, item.dataset ):
append = True
if append:
row = query.item_to_api_value(item)
out.append( self.encode_all_ids( trans, row) )
out.append( self.encode_all_ids( trans, row, True) )
return { 'results' : out }
lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py
+62 -12
View File
@@ -9,6 +9,7 @@ from galaxy.web.base.controller import BaseAPIController
from tool_shed.galaxy_install import repository_util
from tool_shed.util import common_util
from tool_shed.util import encoding_util
import tool_shed.util.shed_util_common as suc
log = logging.getLogger( __name__ )
@@ -179,6 +180,11 @@ class ToolShedRepositoriesController( BaseAPIController ):
# Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain the repository information.
install_repository_dependencies = payload.get( 'install_repository_dependencies', False )
install_tool_dependencies = payload.get( 'install_tool_dependencies', False )
if install_tool_dependencies:
if trans.app.config.tool_dependency_dir is None:
no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set the value of your 'tool_dependency_dir' "
no_tool_dependency_dir_message += "setting in your Galaxy configuration file (universe_wsgi.ini) and restart your Galaxy server."
raise HTTPBadRequest( detail=no_tool_dependency_dir_message )
new_tool_panel_section = payload.get( 'new_tool_panel_section_label', '' )
shed_tool_conf = payload.get( 'shed_tool_conf', None )
if shed_tool_conf:
@@ -211,13 +217,8 @@ class ToolShedRepositoriesController( BaseAPIController ):
tool_path=tool_path,
tool_shed_url=tool_shed_url )
# Create the tool_shed_repository database records and gather additional information for repository installation.
created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \
created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \
repository_util.handle_tool_shed_repositories( trans, installation_dict, using_api=True )
if message and len( repo_info_dicts ) == 1:
# We're attempting to install a single repository that has already been installed into this Galaxy instance.
log.error( message, exc_info=True )
trans.response.status = 500
return dict( status='error', error=message )
if created_or_updated_tool_shed_repositories:
# Build the dictionary of information necessary for installing the repositories.
installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories,
@@ -244,7 +245,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
# order the list of tsr_ids to ensure all repositories install in the required order.
tsr_ids = [ trans.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ]
ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \
repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys )
repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys )
# Install the repositories, keeping track of each one for later display.
for index, tsr_id in enumerate( ordered_tsr_ids ):
tool_shed_repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) )
@@ -266,11 +267,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
action='show',
id=trans.security.encode_id( tool_shed_repository.id ) )
installed_tool_shed_repositories.append( tool_shed_repository_dict )
elif message:
log.error( message, exc_info=True )
trans.response.status = 500
return dict( status='error', error=message )
elif not created_or_updated_tool_shed_repositories and not message:
else:
# We're attempting to install more than 1 repository, and all of them have already been installed.
return dict( status='error', error='All repositories that you are attempting to install have been previously installed.' )
# Display the list of installed repositories.
@@ -362,3 +359,56 @@ class ToolShedRepositoriesController( BaseAPIController ):
elif isinstance( installed_tool_shed_repositories, list ):
all_installed_tool_shed_repositories.extend( installed_tool_shed_repositories )
return all_installed_tool_shed_repositories
@web.expose_api
def repair_repository_revision( self, trans, payload, **kwd ):
"""
POST /api/tool_shed_repositories/repair_repository_revision
Repair a specified repository revision previously installed into Galaxy.
:param key: the current Galaxy admin user's API key
The following parameters are included in the payload.
:param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed
:param name (required): the name of the Repository
:param owner (required): the owner of the Repository
:param changset_revision (required): the changset_revision of the RepositoryMetadata object associated with the Repository
"""
api_key = kwd.get( 'key', None )
# Get the information about the repository to be installed from the payload.
tool_shed_url = payload.get( 'tool_shed_url', '' )
if not tool_shed_url:
raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." )
name = payload.get( 'name', '' )
if not name:
raise HTTPBadRequest( detail="Missing required parameter 'name'." )
owner = payload.get( 'owner', '' )
if not owner:
raise HTTPBadRequest( detail="Missing required parameter 'owner'." )
changeset_revision = payload.get( 'changeset_revision', '' )
if not changeset_revision:
raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." )
tool_shed_repositories = []
tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision )
repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository )
ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] )
ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] )
if ordered_tsr_ids and ordered_repo_info_dicts:
repositories_for_repair = []
for index, tsr_id in enumerate( ordered_tsr_ids ):
repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) )
repo_info_dict = ordered_repo_info_dicts[ index ]
# TODO: handle errors in repair_dict.
repair_dict = repository_util.repair_tool_shed_repository( trans,
repository,
encoding_util.tool_shed_encode( repo_info_dict ) )
repository_dict = repository.get_api_value( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) )
repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
action='show',
id=trans.security.encode_id( repository.id ) )
if repair_dict:
errors = repair_dict.get( repository.name, [] )
repository_dict[ 'errors_attempting_repair' ] = ' '.join( errors )
tool_shed_repositories.append( repository_dict )
# Display the list of repaired repositories.
return tool_shed_repositories
lib/galaxy/webapps/galaxy/api/tools.py
+12 -6
View File
@@ -66,9 +66,9 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
# -- Execute tool. --
# Get tool.
tool_id = payload[ 'tool_id' ]
tool = trans.app.toolbox.get_tool( tool_id )
tool = trans.app.toolbox.get_tool( payload[ 'tool_id' ] ) if 'tool_id' in payload else None
if not tool:
trans.response.status = 404
return { "message": { "type": "error", "text" : trans.app.model.Dataset.conversion_messages.NO_TOOL } }
# Set running history from payload parameters.
@@ -82,7 +82,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
target_history = None
# Set up inputs.
inputs = payload[ 'inputs' ]
inputs = payload.get( 'inputs', {} )
# Find files coming in as multipart file data and add to inputs.
for k, v in payload.iteritems():
if k.startswith("files_"):
@@ -91,17 +91,23 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
# HACK: add run button so that tool.handle_input will run tool.
inputs['runtool_btn'] = 'Execute'
# TODO: encode data ids and decode ids.
# TODO: handle dbkeys
params = util.Params( inputs, sanitize = False )
template, vars = tool.handle_input( trans, params.__dict__, history=target_history)
template, vars = tool.handle_input( trans, params.__dict__, history=target_history )
if 'errors' in vars:
trans.response.status = 400
return { "message": { "type": "error", "data" : vars[ 'errors' ] } }
# TODO: check for errors and ensure that output dataset(s) are available.
output_datasets = vars.get('out_data', {}).values()
output_datasets = vars.get( 'out_data', {} ).values()
rval = {
"outputs": []
}
outputs = rval[ "outputs" ]
#TODO:?? poss. only return ids?
for output in output_datasets:
outputs.append( output.get_api_value() )
output_dict = output.get_api_value()
outputs.append( trans.security.encode_dict_ids( output_dict ) )
return rval
#
lib/galaxy/webapps/galaxy/api/workflows.py
+3 -2
View File
@@ -290,13 +290,14 @@ class WorkflowsAPIController(BaseAPIController, UsesAnnotations):
try:
stored_workflow = trans.sa_session.query(self.app.model.StoredWorkflow).get(trans.security.decode_id(workflow_id))
except Exception,e:
except Exception, e:
trans.response.status = 400
return ("Workflow with ID='%s' can not be found\n Exception: %s") % (workflow_id, str( e ))
# check to see if user has permissions to selected workflow
if stored_workflow.user != trans.user and not trans.user_is_admin():
if trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).filter_by(user=trans.user, stored_workflow=stored_workflow).count() == 0:
trans.response.status = 400
trans.response.status = 403
return("Workflow is not owned by or shared with current user")
#Mark a workflow as deleted
lib/galaxy/webapps/galaxy/buildapp.py
+17 -6
View File
@@ -128,6 +128,10 @@ def app_factory( global_conf, **kwargs ):
name_prefix="workflow_",
path_prefix='/api/workflows/:workflow_id' )
_add_item_provenance_controller( webapp,
name_prefix="history_content_",
path_prefix='/api/histories/:history_id/contents/:history_content_id' )
webapp.mapper.resource( 'dataset', 'datasets', path_prefix='/api' )
webapp.mapper.resource_with_deleted( 'library', 'libraries', path_prefix='/api' )
webapp.mapper.resource( 'sample', 'samples', path_prefix='/api' )
@@ -159,12 +163,13 @@ def app_factory( global_conf, **kwargs ):
webapp.mapper.connect("workflow_dict", '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict(method=['GET']))
# Galaxy API for tool shed features.
webapp.mapper.resource( 'tool_shed_repository',
'tool_shed_repositories',
controller='tool_shed_repositories',
name_prefix='tool_shed_repository_',
path_prefix='/api',
new={ 'install_repository_revision' : 'POST' },
parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) )
'tool_shed_repositories',
member={ 'repair_repository_revision' : 'POST' },
controller='tool_shed_repositories',
name_prefix='tool_shed_repository_',
path_prefix='/api',
new={ 'install_repository_revision' : 'POST' },
parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) )
# Connect logger from app
if app.trace_logger:
webapp.trace_logger = app.trace_logger
@@ -234,6 +239,12 @@ def _add_item_annotation_controller( webapp, name_prefix, path_prefix, **kwd ):
name = "%sannotation" % name_prefix
webapp.mapper.resource(name, "annotation", path_prefix=path_prefix, controller=controller)
def _add_item_provenance_controller( webapp, name_prefix, path_prefix, **kwd ):
controller = "%sprovenance" % name_prefix
name = "%sprovenance" % name_prefix
webapp.mapper.resource(name, "provenance", path_prefix=path_prefix, controller=controller)
def wrap_in_middleware( app, global_conf, **local_conf ):
"""
Based on the configuration wrap `app` in a set of common and useful
lib/galaxy/webapps/galaxy/controllers/admin.py
+2 -1
View File
@@ -682,7 +682,8 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
if email is not None:
user = trans.sa_session.query( trans.app.model.User ).filter_by( email=email ).first()
if user:
trans.set_user( user )
trans.handle_user_logout()
trans.handle_user_login(user)
message = 'You are now logged in as %s, <a target="_top" href="%s">return to the home page</a>' % ( email, url_for( controller='root' ) )
emails = []
else:
lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+327 -219
View File
@@ -73,9 +73,8 @@ class AdminToolshed( AdminGalaxy ):
@web.expose
@web.require_admin
def browse_repository( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] )
return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako',
repository=repository,
@@ -154,10 +153,10 @@ class AdminToolshed( AdminGalaxy ):
@web.expose
@web.require_admin
def browse_tool_dependency( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
tool_dependency = tool_dependency_util.get_tool_dependency( trans, kwd[ 'id' ] )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
tool_dependency_ids = tool_dependency_util.get_tool_dependency_ids( as_string=False, **kwd )
tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_ids[ 0 ] )
if tool_dependency.in_error_state:
message = "This tool dependency is not installed correctly (see the <b>Tool dependency installation error</b> below). "
message += "Choose <b>Uninstall this tool dependency</b> from the <b>Repository Actions</b> menu, correct problems "
@@ -186,9 +185,8 @@ class AdminToolshed( AdminGalaxy ):
@web.expose
@web.require_admin
def browse_tool_sheds( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
return trans.fill_template( '/webapps/galaxy/admin/tool_sheds.mako',
message=message,
status='error' )
@@ -197,8 +195,7 @@ class AdminToolshed( AdminGalaxy ):
@web.require_admin
def check_for_updates( self, trans, **kwd ):
"""Send a request to the relevant tool shed to see if there are any updates."""
params = util.Params( kwd )
repository_id = params.get( 'id', None )
repository_id = kwd.get( 'id', None )
repository = suc.get_installed_tool_shed_repository( trans, repository_id )
tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed )
url = suc.url_join( tool_shed_url,
@@ -215,10 +212,9 @@ class AdminToolshed( AdminGalaxy ):
is because it becomes a bit complex to determine if other installed repositories include tools that require the same entry. For now
we'll never delete entries from config.shed_tool_data_table_config, but we may choose to do so in the future if it becomes necessary.
"""
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
remove_from_disk = params.get( 'remove_from_disk', '' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
remove_from_disk = kwd.get( 'remove_from_disk', '' )
remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk )
tool_shed_repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] )
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository )
@@ -229,7 +225,7 @@ class AdminToolshed( AdminGalaxy ):
else:
repository_install_dir = None
errors = ''
if params.get( 'deactivate_or_uninstall_repository_button', False ):
if kwd.get( 'deactivate_or_uninstall_repository_button', False ):
if tool_shed_repository.includes_tools_for_display_in_tool_panel:
# Handle tool panel alterations.
tool_util.remove_from_tool_panel( trans, tool_shed_repository, shed_tool_conf, uninstall=remove_from_disk_checked )
@@ -321,27 +317,6 @@ class AdminToolshed( AdminGalaxy ):
trans.response.headers['Expires'] = '0'
return suc.get_repository_file_contents( file_path )
@web.expose
@web.require_admin
def get_repository_dependencies( self, trans, repository_id, repository_name, repository_owner, changeset_revision ):
"""
Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined for the received repository
name, owner and changeset revision. The received repository_id is the encoded id of the installed tool shed repository in Galaxy. We
need it so that we can derive the tool shed from which it was installed.
"""
repository = suc.get_installed_tool_shed_repository( trans, repository_id )
tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed )
url = suc.url_join( tool_shed_url,
'repository/get_repository_dependencies?name=%s&owner=%s&changeset_revision=%s' % \
( repository_name, repository_owner, changeset_revision ) )
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url )
if len( raw_text ) > 2:
encoded_text = json.from_json_string( raw_text )
text = encoding_util.tool_shed_decode( encoded_text )
else:
text = ''
return text
@web.expose
@web.require_admin
def get_tool_dependencies( self, trans, repository_id, repository_name, repository_owner, changeset_revision ):
@@ -387,9 +362,8 @@ class AdminToolshed( AdminGalaxy ):
@web.require_admin
def import_workflow( self, trans, workflow_name, repository_id, **kwd ):
"""Import a workflow contained in an installed tool shed repository into the Galaxy instance."""
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
if workflow_name:
workflow_name = encoding_util.tool_shed_decode( workflow_name )
repository = suc.get_tool_shed_repository_by_id( trans, repository_id )
@@ -420,7 +394,8 @@ class AdminToolshed( AdminGalaxy ):
repository_id=repository_id,
changeset_revision=changeset_revision )
# Save the workflow in the Galaxy database.
stored_workflow = workflow_util.save_workflow( trans, workflow )
# Pass workflow_dict along to create annotation at this point
stored_workflow = workflow_util.save_workflow( trans, workflow, workflow_dict )
# Use the latest version of the saved workflow.
workflow = stored_workflow.latest_workflow
if workflow_name:
@@ -461,7 +436,7 @@ class AdminToolshed( AdminGalaxy ):
message = ''
tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository
# Get the tool_dependencies.xml file from the repository.
tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', tool_shed_repository.repo_path( trans.app ) )
tool_dependencies_config = suc.get_config_from_disk( suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, tool_shed_repository.repo_path( trans.app ) )
installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app,
tool_shed_repository=tool_shed_repository,
tool_dependencies_config=tool_dependencies_config,
@@ -485,12 +460,9 @@ class AdminToolshed( AdminGalaxy ):
@web.expose
@web.require_admin
def install_tool_dependencies( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
tool_dependency_ids = util.listify( params.get( 'tool_dependency_ids', None ) )
if not tool_dependency_ids:
tool_dependency_ids = util.listify( params.get( 'id', None ) )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
tool_dependency_ids = tool_dependency_util.get_tool_dependency_ids( as_string=False, **kwd )
tool_dependencies = []
for tool_dependency_id in tool_dependency_ids:
tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_id )
@@ -515,7 +487,7 @@ class AdminToolshed( AdminGalaxy ):
message = 'All of the selected tool dependencies are already installed.'
status = 'error'
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='manage_tool_dependencies',
action='manage_repository_tool_dependencies',
tool_dependency_ids=tool_dependency_ids,
status=status,
message=message ) )
@@ -550,75 +522,12 @@ class AdminToolshed( AdminGalaxy ):
action='monitor_repository_installation',
tool_shed_repository_ids=tsr_ids_for_monitoring ) )
@web.expose
@web.require_admin
def manage_repository( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
repository_id = kwd[ 'id' ]
operation = kwd.get( 'operation', None )
repository = suc.get_installed_tool_shed_repository( trans, repository_id )
if not repository:
return trans.show_error_message( 'Invalid repository specified.' )
tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed )
if repository.status in [ trans.model.ToolShedRepository.installation_status.CLONING ]:
tool_shed_repository_ids = [ repository_id ]
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='monitor_repository_installation',
tool_shed_repository_ids=tool_shed_repository_ids ) )
if repository.can_install and operation == 'install':
# Send a request to the tool shed to install the repository.
url = suc.url_join( tool_shed_url,
'repository/install_repositories_by_revision?name=%s&owner=%s&changeset_revisions=%s&galaxy_url=%s' % \
( repository.name, repository.owner, repository.installed_changeset_revision, ( web.url_for( '/', qualified=True ) ) ) )
return trans.response.send_redirect( url )
description = util.restore_text( params.get( 'description', repository.description ) )
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository )
if relative_install_dir:
repo_files_dir = os.path.abspath( os.path.join( tool_path, relative_install_dir, repository.name ) )
else:
repo_files_dir = None
if repository.in_error_state:
message = "This repository is not installed correctly (see the <b>Repository installation error</b> below). Choose "
message += "<b>Reset to install</b> from the <b>Repository Actions</b> menu, correct problems if necessary and try "
message += "installing the repository again."
status = "error"
elif repository.can_install:
message = "This repository is not installed. You can install it by choosing <b>Install</b> from the <b>Repository Actions</b> menu."
status = "error"
elif params.get( 'edit_repository_button', False ):
if description != repository.description:
repository.description = description
trans.sa_session.add( repository )
trans.sa_session.flush()
message = "The repository information has been updated."
containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( trans=trans,
tool_shed_url=tool_shed_url,
tool_path=tool_path,
repository=repository,
reinstalling=False,
required_repo_info_dicts=None )
return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
repository=repository,
description=description,
repo_files_dir=repo_files_dir,
containers_dict=containers_dict,
message=message,
status=status )
@web.expose
@web.require_admin
def manage_repositories( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
tsrid = params.get( 'tool_shed_repository_id', None )
tsridslist = util.listify( params.get( 'tool_shed_repository_ids', None ) )
if not tsridslist:
tsridslist = util.listify( params.get( 'id', None ) )
if tsrid and tsrid not in tsridslist:
tsridslist.append( tsrid )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
tsridslist = repository_util.get_tool_shed_repository_ids( **kwd )
if 'operation' in kwd:
operation = kwd[ 'operation' ].lower()
if not tsridslist:
@@ -649,7 +558,7 @@ class AdminToolshed( AdminGalaxy ):
kwd[ 'message' ] = 'All selected tool shed repositories are already uninstalled.'
kwd[ 'status' ] = 'error'
elif operation == "install":
reinstalling = util.string_as_bool( params.get( 'reinstalling', False ) )
reinstalling = util.string_as_bool( kwd.get( 'reinstalling', False ) )
encoded_kwd = kwd[ 'encoded_kwd' ]
decoded_kwd = encoding_util.tool_shed_decode( encoded_kwd )
tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ]
@@ -661,7 +570,7 @@ class AdminToolshed( AdminGalaxy ):
# Some repositories may have repository dependencies that are required to be installed before the dependent repository, so we'll
# order the list of tsr_ids to ensure all repositories install in the required order.
ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \
repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys )
repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys )
for tsr_id in ordered_tsr_ids:
repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) )
if repository.status in [ trans.model.ToolShedRepository.installation_status.NEW,
@@ -684,36 +593,75 @@ class AdminToolshed( AdminGalaxy ):
@web.expose
@web.require_admin
def manage_tool_dependencies( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
def manage_repository( self, trans, **kwd ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository_id = kwd[ 'id' ]
operation = kwd.get( 'operation', None )
repository = suc.get_installed_tool_shed_repository( trans, repository_id )
if not repository:
return trans.show_error_message( 'Invalid repository specified.' )
tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed )
if repository.status in [ trans.model.ToolShedRepository.installation_status.CLONING ]:
tool_shed_repository_ids = [ repository_id ]
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='monitor_repository_installation',
tool_shed_repository_ids=tool_shed_repository_ids ) )
if repository.can_install and operation == 'install':
# Send a request to the tool shed to install the repository.
url = suc.url_join( tool_shed_url,
'repository/install_repositories_by_revision?name=%s&owner=%s&changeset_revisions=%s&galaxy_url=%s' % \
( repository.name, repository.owner, repository.installed_changeset_revision, ( web.url_for( '/', qualified=True ) ) ) )
return trans.response.send_redirect( url )
description = kwd.get( 'description', repository.description )
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository )
if relative_install_dir:
repo_files_dir = os.path.abspath( os.path.join( tool_path, relative_install_dir, repository.name ) )
else:
repo_files_dir = None
if repository.in_error_state:
message = "This repository is not installed correctly (see the <b>Repository installation error</b> below). Choose "
message += "<b>Reset to install</b> from the <b>Repository Actions</b> menu, correct problems if necessary and try "
message += "installing the repository again."
status = "error"
elif repository.can_install:
message = "This repository is not installed. You can install it by choosing <b>Install</b> from the <b>Repository Actions</b> menu."
status = "error"
elif kwd.get( 'edit_repository_button', False ):
if description != repository.description:
repository.description = description
trans.sa_session.add( repository )
trans.sa_session.flush()
message = "The repository information has been updated."
containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( trans=trans,
tool_shed_url=tool_shed_url,
tool_path=tool_path,
repository=repository,
reinstalling=False,
required_repo_info_dicts=None )
return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
repository=repository,
description=description,
repo_files_dir=repo_files_dir,
containers_dict=containers_dict,
message=message,
status=status )
@web.expose
@web.require_admin
def manage_repository_tool_dependencies( self, trans, **kwd ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
tool_dependency_ids = tool_dependency_util.get_tool_dependency_ids( as_string=False, **kwd )
# We need a tool_shed_repository, so get it from one of the tool_dependencies.
tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_ids[ 0 ] )
tool_shed_repository = tool_dependency.tool_shed_repository
self.tool_dependency_grid.title = "Tool shed repository '%s' tool dependencies" % tool_shed_repository.name
self.tool_dependency_grid.global_actions = \
[ grids.GridAction( label='Manage repository',
url_args=dict( controller='admin_toolshed',
action='manage_repository',
id=trans.security.encode_id( tool_shed_repository.id ) ) ),
grids.GridAction( label='Browse repository',
url_args=dict( controller='admin_toolshed',
action='browse_repository',
id=trans.security.encode_id( tool_shed_repository.id ) ) ),
grids.GridAction( label='Get repository updates',
url_args=dict( controller='admin_toolshed',
action='check_for_updates',
id=trans.security.encode_id( tool_shed_repository.id ) ) ),
grids.GridAction( label='Set tool versions',
url_args=dict( controller='admin_toolshed',
action='set_tool_versions',
id=trans.security.encode_id( tool_shed_repository.id ) ) ),
grids.GridAction( label='Deactivate or uninstall repository',
url_args=dict( controller='admin_toolshed',
action='deactivate_or_uninstall_repository',
id=trans.security.encode_id( tool_shed_repository.id ) ) ) ]
if tool_dependency_ids:
# We need a tool_shed_repository, so get it from one of the tool_dependencies.
tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_ids[ 0 ] )
tool_shed_repository = tool_dependency.tool_shed_repository
else:
# The user must be on the manage_repository_tool_dependencies page and clicked the button to either install or uninstall a
# tool dependency, but they didn't check any of the available tool dependencies on which to perform the action.
repository_id = kwd.get( 'repository_id', None )
tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, repository_id )
if 'operation' in kwd:
operation = kwd[ 'operation' ].lower()
if not tool_dependency_ids:
@@ -722,7 +670,7 @@ class AdminToolshed( AdminGalaxy ):
kwd[ 'status' ] = 'error'
del kwd[ 'operation' ]
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='manage_tool_dependencies',
action='manage_repository_tool_dependencies',
**kwd ) )
if operation == 'browse':
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
@@ -759,21 +707,88 @@ class AdminToolshed( AdminGalaxy ):
message = 'Set the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file (universe_wsgi.ini) '
message += ' and restart your Galaxy server to install tool dependencies.'
kwd[ 'message' ] = message
kwd[ 'status' ] = 'error'
return self.tool_dependency_grid( trans, **kwd )
kwd[ 'status' ] = 'error'
installed_tool_dependencies_select_field = suc.build_tool_dependencies_select_field( trans,
tool_shed_repository=tool_shed_repository,
name='inst_td_ids',
uninstalled=False )
uninstalled_tool_dependencies_select_field = suc.build_tool_dependencies_select_field( trans,
tool_shed_repository=tool_shed_repository,
name='uninstalled_tool_dependency_ids',
uninstalled=True )
return trans.fill_template( '/admin/tool_shed_repository/manage_repository_tool_dependencies.mako',
repository=tool_shed_repository,
installed_tool_dependencies_select_field=installed_tool_dependencies_select_field,
uninstalled_tool_dependencies_select_field=uninstalled_tool_dependencies_select_field,
message=message,
status=status )
@web.expose
@web.require_admin
def manage_tool_dependencies( self, trans, **kwd ):
# This method is called when tool dependencies are being installed. See the related manage_repository_tool_dependencies
# method for managing the tool dependencies for a specified installed tool shed repository.
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
tool_dependency_ids = tool_dependency_util.get_tool_dependency_ids( as_string=False, **kwd )
repository_id = kwd.get( 'repository_id', None )
if tool_dependency_ids:
# We need a tool_shed_repository, so get it from one of the tool_dependencies.
tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_ids[ 0 ] )
tool_shed_repository = tool_dependency.tool_shed_repository
else:
# The user must be on the manage_repository_tool_dependencies page and clicked the button to either install or uninstall a
# tool dependency, but they didn't check any of the available tool dependencies on which to perform the action.
tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, repository_id )
self.tool_dependency_grid.title = "Tool shed repository '%s' tool dependencies" % tool_shed_repository.name
if 'operation' in kwd:
operation = kwd[ 'operation' ].lower()
if not tool_dependency_ids:
message = 'Select at least 1 tool dependency to %s.' % operation
kwd[ 'message' ] = message
kwd[ 'status' ] = 'error'
del kwd[ 'operation' ]
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='manage_tool_dependencies',
**kwd ) )
if operation == 'browse':
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='browse_tool_dependency',
**kwd ) )
elif operation == "install":
if trans.app.config.tool_dependency_dir:
tool_dependencies_for_installation = []
for tool_dependency_id in tool_dependency_ids:
tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_id )
if tool_dependency.status in [ trans.model.ToolDependency.installation_status.NEVER_INSTALLED,
trans.model.ToolDependency.installation_status.UNINSTALLED ]:
tool_dependencies_for_installation.append( tool_dependency )
if tool_dependencies_for_installation:
self.initiate_tool_dependency_installation( trans, tool_dependencies_for_installation )
else:
kwd[ 'message' ] = 'All selected tool dependencies are already installed.'
kwd[ 'status' ] = 'error'
else:
message = 'Set the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file (universe_wsgi.ini) '
message += ' and restart your Galaxy server to install tool dependencies.'
kwd[ 'message' ] = message
kwd[ 'status' ] = 'error'
# Redirect if no tool dependencies are in the process of being installed.
if tool_shed_repository.tool_dependencies_being_installed:
return self.tool_dependency_grid( trans, **kwd )
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='manage_repository_tool_dependencies',
tool_dependency_ids=tool_dependency_ids,
repository_id=repository_id,
message=message,
status=status ) )
@web.expose
@web.require_admin
def monitor_repository_installation( self, trans, **kwd ):
params = util.Params( kwd )
tsrid = params.get( 'tool_shed_repository_id', None )
tsridslist = util.listify( params.get( 'tool_shed_repository_ids', None ) )
tsridslist = repository_util.get_tool_shed_repository_ids( **kwd )
if not tsridslist:
tsridslist = util.listify( params.get( 'id', None ) )
if tsrid and tsrid not in tsridslist:
tsridslist.append( tsrid )
if not tsridslist:
tsridslist = get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False )
tsridslist = suc.get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False )
kwd[ 'tool_shed_repository_ids' ] = tsridslist
return self.repository_installation_grid( trans, **kwd )
@@ -823,11 +838,17 @@ class AdminToolshed( AdminGalaxy ):
'repository/get_repository_information?repository_ids=%s&changeset_revisions=%s' % \
( repository_ids, changeset_revisions ) )
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url )
repo_information_dict = json.from_json_string( raw_text )
includes_tools = util.string_as_bool( repo_information_dict.get( 'includes_tools', False ) )
includes_tools_for_display_in_tool_panel = util.string_as_bool( repo_information_dict.get( 'includes_tools_for_display_in_tool_panel', False ) )
has_repository_dependencies = util.string_as_bool( repo_information_dict.get( 'has_repository_dependencies', False ) )
includes_tool_dependencies = util.string_as_bool( repo_information_dict.get( 'includes_tool_dependencies', False ) )
repo_information_dict = json.from_json_string( raw_text )
for encoded_repo_info_dict in repo_information_dict.get( 'repo_info_dicts', [] ):
decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict )
if not includes_tools:
includes_tools = util.string_as_bool( decoded_repo_info_dict.get( 'includes_tools', False ) )
if not includes_tools_for_display_in_tool_panel:
includes_tools_for_display_in_tool_panel = util.string_as_bool( decoded_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False ) )
if not has_repository_dependencies:
has_repository_dependencies = util.string_as_bool( repo_information_dict.get( 'has_repository_dependencies', False ) )
if not includes_tool_dependencies:
includes_tool_dependencies = util.string_as_bool( repo_information_dict.get( 'includes_tool_dependencies', False ) )
encoded_repo_info_dicts = util.listify( repo_information_dict.get( 'repo_info_dicts', [] ) )
repo_info_dicts = [ encoding_util.tool_shed_decode( encoded_repo_info_dict ) for encoded_repo_info_dict in encoded_repo_info_dicts ]
if ( not includes_tools_for_display_in_tool_panel and kwd.get( 'select_shed_tool_panel_config_button', False ) ) or \
@@ -846,15 +867,8 @@ class AdminToolshed( AdminGalaxy ):
tool_panel_section=tool_panel_section,
tool_path=tool_path,
tool_shed_url=tool_shed_url )
created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \
created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \
repository_util.handle_tool_shed_repositories( trans, installation_dict, using_api=False )
if message and len( repo_info_dicts ) == 1:
# We're undoubtedly attempting to install a repository that has been previously installed.
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='browse_repositories',
message=message,
status='error' ) )
if created_or_updated_tool_shed_repositories:
installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories,
filtered_repo_info_dicts=filtered_repo_info_dicts,
@@ -891,9 +905,25 @@ class AdminToolshed( AdminGalaxy ):
if len( repo_info_dicts ) == 1:
# If we're installing a single repository, see if it contains a readme or dependencies that we can display.
repo_info_dict = repo_info_dicts[ 0 ]
name, repository_owner, changeset_revision, includes_tool_dependencies, installed_repository_dependencies, \
missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies = \
common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies )
dependencies_for_repository_dict = common_install_util.get_dependencies_for_repository( trans,
tool_shed_url,
repo_info_dict,
includes_tool_dependencies )
changeset_revision = dependencies_for_repository_dict.get( 'changeset_revision', None )
if not has_repository_dependencies:
has_repository_dependencies = dependencies_for_repository_dict.get( 'has_repository_dependencies', False )
if not includes_tool_dependencies:
includes_tool_dependencies = dependencies_for_repository_dict.get( 'includes_tool_dependencies', False )
if not includes_tools:
includes_tools = dependencies_for_repository_dict.get( 'includes_tools', False )
if not includes_tools_for_display_in_tool_panel:
includes_tools_for_display_in_tool_panel = dependencies_for_repository_dict.get( 'includes_tools_for_display_in_tool_panel', False )
installed_repository_dependencies = dependencies_for_repository_dict.get( 'installed_repository_dependencies', None )
installed_tool_dependencies = dependencies_for_repository_dict.get( 'installed_tool_dependencies', None )
missing_repository_dependencies = dependencies_for_repository_dict.get( 'missing_repository_dependencies', None )
missing_tool_dependencies = dependencies_for_repository_dict.get( 'missing_tool_dependencies', None )
name = dependencies_for_repository_dict.get( 'name', None )
repository_owner = dependencies_for_repository_dict.get( 'repository_owner', None )
readme_files_dict = readme_util.get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict )
# We're handling 1 of 2 scenarios here: (1) we're installing a tool shed repository for the first time, so we've retrieved the list of installed
# and missing repository dependencies from the database (2) we're handling the scenario where an error occurred during the installation process,
@@ -912,9 +942,25 @@ class AdminToolshed( AdminGalaxy ):
# We're installing a list of repositories, each of which may have tool dependencies or repository dependencies.
containers_dicts = []
for repo_info_dict in repo_info_dicts:
name, repository_owner, changeset_revision, includes_tool_dependencies, installed_repository_dependencies, \
missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies = \
common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies )
dependencies_for_repository_dict = common_install_util.get_dependencies_for_repository( trans,
tool_shed_url,
repo_info_dict,
includes_tool_dependencies )
changeset_revision = dependencies_for_repository_dict.get( 'changeset_revision', None )
if not has_repository_dependencies:
has_repository_dependencies = dependencies_for_repository_dict.get( 'has_repository_dependencies', False )
if not includes_tool_dependencies:
includes_tool_dependencies = dependencies_for_repository_dict.get( 'includes_tool_dependencies', False )
if not includes_tools:
includes_tools = dependencies_for_repository_dict.get( 'includes_tools', False )
if not includes_tools_for_display_in_tool_panel:
includes_tools_for_display_in_tool_panel = dependencies_for_repository_dict.get( 'includes_tools_for_display_in_tool_panel', False )
installed_repository_dependencies = dependencies_for_repository_dict.get( 'installed_repository_dependencies', None )
installed_tool_dependencies = dependencies_for_repository_dict.get( 'installed_tool_dependencies', None )
missing_repository_dependencies = dependencies_for_repository_dict.get( 'missing_repository_dependencies', None )
missing_tool_dependencies = dependencies_for_repository_dict.get( 'missing_tool_dependencies', None )
name = dependencies_for_repository_dict.get( 'name', None )
repository_owner = dependencies_for_repository_dict.get( 'repository_owner', None )
containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans,
tool_shed_url=tool_shed_url,
tool_path=tool_path,
@@ -1033,11 +1079,7 @@ class AdminToolshed( AdminGalaxy ):
else:
# Entering this else block occurs only if the tool_shed_repository does not include any valid tools.
if install_repository_dependencies:
repository_dependencies = self.get_repository_dependencies( trans=trans,
repository_id=repository_id,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
changeset_revision=tool_shed_repository.changeset_revision )
repository_dependencies = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, tool_shed_repository )
else:
repository_dependencies = None
if metadata:
@@ -1056,7 +1098,7 @@ class AdminToolshed( AdminGalaxy ):
repository_dependencies=repository_dependencies )
repo_info_dicts.append( repo_info_dict )
# Make sure all tool_shed_repository records exist.
created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \
created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \
repository_dependency_util.create_repository_dependency_objects( trans=trans,
tool_path=tool_path,
tool_shed_url=tool_shed_url,
@@ -1103,6 +1145,52 @@ class AdminToolshed( AdminGalaxy ):
initiate_repository_installation_ids=encoded_repository_ids,
reinstalling=True )
@web.expose
@web.require_admin
def repair_repository( self, trans, **kwd ):
"""
Inspect the repository dependency hierarchy for a specified repository and attempt to make sure they are all properly installed as well as
each repository's tool dependencies.
"""
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
if not repository_id:
message = 'Invalid installed tool shed repository id %s received.' % str( repository_id )
status = 'error'
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='browse_repositories',
message=message,
status=status ) )
tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id )
if kwd.get( 'repair_repository_button', False ):
encoded_repair_dict = kwd.get( 'repair_dict', None )
if encoded_repair_dict:
repair_dict = encoding_util.tool_shed_decode( encoded_repair_dict )
else:
repair_dict = None
if not repair_dict:
repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository )
ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] )
ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] )
if ordered_tsr_ids and ordered_repo_info_dicts:
repositories_for_repair = []
for tsr_id in ordered_tsr_ids:
repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) )
repositories_for_repair.append( repository )
return self.repair_tool_shed_repositories( trans, repositories_for_repair, ordered_repo_info_dicts )
tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id )
repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository )
encoded_repair_dict = encoding_util.tool_shed_encode( repair_dict )
ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] )
ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] )
return trans.fill_template( 'admin/tool_shed_repository/repair_repository.mako',
repository=tool_shed_repository,
encoded_repair_dict=encoded_repair_dict,
repair_dict=repair_dict,
message=message,
status=status )
@web.json
def repository_installation_status_updates( self, trans, ids=None, status_list=None ):
# Avoid caching
@@ -1124,6 +1212,21 @@ class AdminToolshed( AdminGalaxy ):
'utf-8' ) ) )
return rval
@web.expose
@web.require_admin
def repair_tool_shed_repositories( self, trans, tool_shed_repositories, repo_info_dicts, **kwd ):
"""Repair specified tool shed repositories."""
# The received lists of tool_shed_repositories and repo_info_dicts are ordered.
for index, tool_shed_repository in enumerate( tool_shed_repositories ):
repo_info_dict = repo_info_dicts[ index ]
repair_dict = repository_util.repair_tool_shed_repository( trans,
tool_shed_repository,
encoding_util.tool_shed_encode( repo_info_dict ) )
tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ]
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='monitor_repository_installation',
tool_shed_repository_ids=tsr_ids_for_monitoring ) )
@web.expose
@web.require_admin
def reselect_tool_panel_section( self, trans, **kwd ):
@@ -1192,11 +1295,7 @@ class AdminToolshed( AdminGalaxy ):
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url )
readme_files_dict = json.from_json_string( raw_text )
tool_dependencies = metadata.get( 'tool_dependencies', None )
repository_dependencies = self.get_repository_dependencies( trans=trans,
repository_id=repository_id,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
changeset_revision=tool_shed_repository.changeset_revision )
repository_dependencies = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, tool_shed_repository )
repo_info_dict = repository_util.create_repo_info_dict( trans=trans,
repository_clone_url=repository_clone_url,
changeset_revision=tool_shed_repository.changeset_revision,
@@ -1207,9 +1306,22 @@ class AdminToolshed( AdminGalaxy ):
repository_metadata=None,
tool_dependencies=tool_dependencies,
repository_dependencies=repository_dependencies )
repository_name, repository_owner, changeset_revision, includes_tool_dependencies, installed_repository_dependencies, \
missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies = \
common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies )
dependencies_for_repository_dict = common_install_util.get_dependencies_for_repository( trans,
tool_shed_url,
repo_info_dict,
includes_tool_dependencies )
changeset_revision = dependencies_for_repository_dict.get( 'changeset_revision', None )
has_repository_dependencies = dependencies_for_repository_dict.get( 'has_repository_dependencies', False )
includes_tool_dependencies = dependencies_for_repository_dict.get( 'includes_tool_dependencies', False )
includes_tools = dependencies_for_repository_dict.get( 'includes_tools', False )
includes_tools_for_display_in_tool_panel = dependencies_for_repository_dict.get( 'includes_tools_for_display_in_tool_panel', False )
installed_repository_dependencies = dependencies_for_repository_dict.get( 'installed_repository_dependencies', None )
installed_tool_dependencies = dependencies_for_repository_dict.get( 'installed_tool_dependencies', None )
missing_repository_dependencies = dependencies_for_repository_dict.get( 'missing_repository_dependencies', None )
missing_tool_dependencies = dependencies_for_repository_dict.get( 'missing_tool_dependencies', None )
repository_name = dependencies_for_repository_dict.get( 'name', None )
repository_owner = dependencies_for_repository_dict.get( 'repository_owner', None )
if installed_repository_dependencies or missing_repository_dependencies:
has_repository_dependencies = True
else:
@@ -1297,7 +1409,7 @@ class AdminToolshed( AdminGalaxy ):
if 'reset_metadata_on_selected_repositories_button' in kwd:
message, status = metadata_util.reset_metadata_on_selected_repositories( trans, **kwd )
else:
message = util.restore_text( kwd.get( 'message', '' ) )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repositories_select_field = suc.build_repository_ids_select_field( trans )
return trans.fill_template( '/admin/tool_shed_repository/reset_metadata_on_selected_repositories.mako',
@@ -1350,13 +1462,13 @@ class AdminToolshed( AdminGalaxy ):
"""An error occurred while cloning the repository, so reset everything necessary to enable another attempt."""
repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] )
if kwd.get( 'reset_repository', False ):
suc.set_repository_attributes( trans,
repository,
status=trans.model.ToolShedRepository.installation_status.NEW,
error_message=None,
deleted=False,
uninstalled=False,
remove_from_disk=True )
repository_util.set_repository_attributes( trans,
repository,
status=trans.model.ToolShedRepository.installation_status.NEW,
error_message=None,
deleted=False,
uninstalled=False,
remove_from_disk=True )
new_kwd = {}
new_kwd[ 'message' ] = "You can now attempt to install the repository named <b>%s</b> again." % repository.name
new_kwd[ 'status' ] = "done"
@@ -1430,12 +1542,11 @@ class AdminToolshed( AdminGalaxy ):
@web.expose
@web.require_admin
def uninstall_tool_dependencies( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
tool_dependency_ids = util.listify( params.get( 'tool_dependency_ids', None ) )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
tool_dependency_ids = tool_dependency_util.get_tool_dependency_ids( as_string=False, **kwd )
if not tool_dependency_ids:
tool_dependency_ids = util.listify( params.get( 'id', None ) )
tool_dependency_ids = util.listify( kwd.get( 'id', None ) )
tool_dependencies = []
for tool_dependency_id in tool_dependency_ids:
tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_id )
@@ -1460,7 +1571,7 @@ class AdminToolshed( AdminGalaxy ):
message = "These tool dependencies have been uninstalled: %s" % ','.join( td.name for td in tool_dependencies_for_uninstallation )
td_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies ]
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='manage_tool_dependencies',
action='manage_repository_tool_dependencies',
tool_dependency_ids=td_ids,
status=status,
message=message ) )
@@ -1475,15 +1586,14 @@ class AdminToolshed( AdminGalaxy ):
@web.require_admin
def update_to_changeset_revision( self, trans, **kwd ):
"""Update a cloned repository to the latest revision possible."""
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
tool_shed_url = kwd[ 'tool_shed_url' ]
name = params.get( 'name', None )
owner = params.get( 'owner', None )
changeset_revision = params.get( 'changeset_revision', None )
latest_changeset_revision = params.get( 'latest_changeset_revision', None )
latest_ctx_rev = params.get( 'latest_ctx_rev', None )
name = kwd.get( 'name', None )
owner = kwd.get( 'owner', None )
changeset_revision = kwd.get( 'changeset_revision', None )
latest_changeset_revision = kwd.get( 'latest_changeset_revision', None )
latest_ctx_rev = kwd.get( 'latest_ctx_rev', None )
repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision )
if changeset_revision and latest_changeset_revision and latest_ctx_rev:
if changeset_revision == latest_changeset_revision:
@@ -1567,9 +1677,8 @@ class AdminToolshed( AdminGalaxy ):
@web.expose
@web.require_admin
def view_tool_metadata( self, trans, repository_id, tool_id, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository = suc.get_installed_tool_shed_repository( trans, repository_id )
repository_metadata = repository.metadata
shed_config_dict = repository.get_shed_config_dict( trans.app )
@@ -1600,9 +1709,8 @@ class AdminToolshed( AdminGalaxy ):
@web.require_admin
def view_workflow( self, trans, workflow_name=None, repository_id=None, **kwd ):
"""Retrieve necessary information about a workflow from the database so that it can be displayed in an svg image."""
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
if workflow_name:
workflow_name = encoding_util.tool_shed_decode( workflow_name )
repository = suc.get_tool_shed_repository_by_id( trans, repository_id )
lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py
+14 -5
View File
@@ -35,8 +35,12 @@ class CloudController(BaseUIController):
BaseUIController.__init__(self, app)
@web.expose
def index(self, trans, share_string=None):
return trans.fill_template("cloud/index.mako", default_keypair = DEFAULT_KEYPAIR, share_string=share_string)
def index(self, trans, share_string=None, ami=None, bucket_default = None):
return trans.fill_template("cloud/index.mako",
default_keypair = DEFAULT_KEYPAIR,
share_string=share_string,
ami=ami,
bucket_default=bucket_default)
@web.expose
def get_account_info(self, trans, key_id, secret, **kwargs):
@@ -92,7 +96,7 @@ class CloudController(BaseUIController):
return to_json_string(account_info)
@web.expose
def launch_instance(self, trans, cluster_name, password, key_id, secret, instance_type, share_string, keypair, zone=None, **kwargs):
def launch_instance(self, trans, cluster_name, password, key_id, secret, instance_type, share_string, keypair, ami=DEFAULT_AMI, zone=None, bucket_default=None, **kwargs):
ec2_error = None
try:
# Create security group & key pair used when starting an instance
@@ -113,11 +117,16 @@ class CloudController(BaseUIController):
user_provided_data['password'] = password
if share_string:
user_provided_data['share_string'] = share_string
if bucket_default:
user_provided_data['bucket_default'] = bucket_default
rs = run_instance(ec2_conn=ec2_conn,
image_id = ami,
user_provided_data=user_provided_data,
key_name=kp_name,
security_groups=[sg_name],
placement=zone)
placement=zone
)
if rs:
instance = rs.instances[0]
ct = 0
@@ -266,7 +275,7 @@ def create_key_pair(ec2_conn, key_name=DEFAULT_KEYPAIR):
return None, None
return kp.name, kp.material
def run_instance(ec2_conn, user_provided_data, image_id=DEFAULT_AMI,
def run_instance(ec2_conn, user_provided_data, image_id=None,
kernel_id=None, ramdisk_id=None, key_name=DEFAULT_KEYPAIR,
placement=None, security_groups=['CloudMan']):
""" Start an instance. If instance start was OK, return the ResultSet object
lib/galaxy/webapps/galaxy/controllers/library_common.py
+3
View File
@@ -949,7 +949,10 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin ):
# Send list of data formats to the upload form so the "extension" select list can be populated dynamically
file_formats = trans.app.datatypes_registry.upload_file_formats
dbkeys = trans.app.genomes.get_dbkeys( trans )
dbkeys.sort( key=lambda dbkey: dbkey[0].lower() )
# Send the current history to the form to enable importing datasets from history to library
history = trans.get_history()
if history is not None:
lib/galaxy/webapps/galaxy/controllers/root.py
+5 -91
View File
@@ -39,27 +39,6 @@ class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAsso
params=kwd )
## ---- Tool related -----------------------------------------------------
@web.expose
def tool_menu( self, trans ):
"""Renders the tool panel of the Galaxy UI.
"""
if trans.app.config.require_login and not trans.user:
return trans.fill_template( '/no_access.mako', message='Please log in to access Galaxy tools.' )
toolbox = self.get_toolbox()
## Get most recently used tools.
# recent_tools = []
# if trans.user:
# for row in trans.sa_session.query( self.app.model.Job.tool_id ) \
# .filter( self.app.model.Job.user == trans.user ) \
# .order_by( self.app.model.Job.create_time.desc() ):
# tool_id = row[0]
# a_tool = toolbox.get_tool( tool_id )
# if a_tool and not a_tool.hidden and a_tool not in recent_tools:
# recent_tools.append( a_tool )
# ## TODO: make number of recently used tools a user preference.
# if len( recent_tools ) == 5:
# break
return trans.fill_template( '/root/tool_menu.mako', toolbox=toolbox )
@web.json
def tool_search( self, trans, **kwd ):
@@ -178,76 +157,6 @@ class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAsso
history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ),
show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status )
@web.expose
def profile_history( self, trans, as_xml=False, show_deleted=None, show_hidden=None, hda_id=None, **kwd ):
"""
Same as above but adds SimpleProfiler to get some
profiling times for the operations done.
"""
if as_xml:
return self.history_as_xml( trans,
show_deleted=string_as_bool( show_deleted ), show_hidden=string_as_bool( show_hidden ) )
# get all datasets server-side, client-side will get flags and render appropriately
show_deleted = string_as_bool_or_none( show_deleted )
show_purged = show_deleted
show_hidden = string_as_bool_or_none( show_hidden )
params = Params( kwd )
message = params.get( 'message', '' )
#TODO: ugh...
message = message if message != 'None' else ''
status = params.get( 'status', 'done' )
if trans.app.config.require_login and not trans.user:
return trans.fill_template( '/no_access.mako', message = 'Please log in to access Galaxy histories.' )
def err_msg( where=None ):
where = where if where else 'getting the history data from the server'
err_msg = ( 'An error occurred %s. '
+ 'Please contact a Galaxy administrator if the problem persists.' ) %( where )
return err_msg, 'error'
profiler = SimpleProfiler()
profiler.start()
history_dictionary = {}
hda_dictionaries = []
import pprint
try:
history = trans.get_history( create=True )
profiler.report( 'trans.get_history' )
hdas = self.get_history_datasets( trans, history,
show_deleted=True, show_hidden=True, show_purged=True )
profiler.report( 'get_history_datasets' )
for hda in hdas:
try:
( hda_profiler, hda_dict ) = self.profile_get_hda_dict( trans, hda )
profiler.reports.extend( hda_profiler.get_reports() )
profiler.report( '\t hda -> dictionary (%s)' %( hda.name ) )
hda_dictionaries.append( hda_dict )
except Exception, exc:
# don't fail entire list if hda err's, record and move on
log.error( 'Error bootstrapping hda %d: %s', hda.id, str( exc ), exc_info=True )
hda_dictionaries.append( self.get_hda_dict_with_error( trans, hda, str( exc ) ) )
profiler.report( 'hdas -> dictionaries' )
# re-use the hdas above to get the history data...
history_dictionary = self.get_history_dict( trans, history, hda_dictionaries=hda_dictionaries )
profiler.report( 'history -> dictionary' )
except Exception, exc:
user_id = str( trans.user.id ) if trans.user else '(anonymous)'
log.error( 'Error bootstrapping history for user %s: %s', user_id, str( exc ), exc_info=True )
message, status = err_msg()
history_dictionary[ 'error' ] = message
return trans.stream_template_mako( "root/history.mako",
history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ),
show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status,
profiling=profiler.get_reports() )
## ---- Dataset display / editing ----------------------------------------
@web.expose
def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ):
@@ -546,6 +455,11 @@ class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAsso
except:
return trans.show_error_message( "<p>Failed to make secondary dataset primary.</p>" )
@web.expose
def welcome( self, trans ):
welcome_url = trans.app.config.welcome_url
return trans.response.send_redirect( url_for( welcome_url ) )
@web.expose
def bucket_proxy( self, trans, bucket=None, **kwd):
if bucket:
lib/galaxy/webapps/galaxy/controllers/tool_runner.py
+10 -2
View File
@@ -52,7 +52,7 @@ class ToolRunner( BaseUIController ):
def index(self, trans, tool_id=None, from_noframe=None, **kwd):
# No tool id passed, redirect to main page
if tool_id is None:
return trans.response.send_redirect( url_for( "/static/welcome.html" ) )
return trans.response.send_redirect( url_for( controller="root", action="welcome" ) )
tool_version_select_field, tools, tool = self.__get_tool_components( tool_id,
tool_version=None,
get_loaded_tools_by_lineage=False,
@@ -218,6 +218,14 @@ class ToolRunner( BaseUIController ):
# Create a fake tool_state for the tool, with the parameters values
state = tool.new_state( trans )
state.inputs = params_objects
# If the job failed and has dependencies, allow dependency remap
if job.state == job.states.ERROR:
try:
if [ hda.dependent_jobs for hda in [ jtod.dataset for jtod in job.output_datasets ] if hda.dependent_jobs ]:
state.rerun_remap_job_id = trans.app.security.encode_id(job.id)
except:
# Job has no outputs?
pass
#create an incoming object from the original job's dataset-modified param objects
incoming = {}
params_to_incoming( incoming, tool.inputs, params_objects, trans.app )
@@ -287,7 +295,7 @@ class ToolRunner( BaseUIController ):
tool_state.decode( encoded_state, tool, trans.app )
else:
tool_state = tool.new_state( trans )
errors = tool.update_state( trans, tool.inputs, tool_state.inputs, kwd, update_only = True )
tool.update_state( trans, tool.inputs, tool_state.inputs, kwd, update_only = True )
datasets = []
dataset_upload_inputs = []
for input_name, input in tool.inputs.iteritems():
lib/galaxy/webapps/galaxy/controllers/user.py
+1 -1
View File
@@ -492,7 +492,7 @@ class User( BaseUIController, UsesFormDefinitionsMixin ):
message = 'You are now logged in as %s.<br>You can <a target="_top" href="%s">go back to the page you were visiting</a> or <a target="_top" href="%s">go to the home page</a>.' % \
( user.email, redirect, url_for( '/' ) )
if trans.app.config.require_login:
message += ' <a target="_top" href="%s">Click here</a> to continue to the home page.' % web.url_for( '/static/welcome.html' )
message += ' <a target="_top" href="%s">Click here</a> to continue to the home page.' % web.url_for( controller="root", action="welcome" )
success = True
return ( message, status, user, success )
lib/galaxy/webapps/galaxy/controllers/visualization.py
-2
View File
@@ -848,7 +848,6 @@ class VisualizationController( BaseUIController, SharableMixin, UsesAnnotations,
get the visualization with the given id; otherwise, create a new visualization using
a given dataset and regions.
"""
print 'sweepster:', id, hda_ldda, dataset_id, regions
regions = regions or '{}'
# Need to create history if necessary in order to create tool form.
trans.get_history( create=True )
@@ -861,7 +860,6 @@ class VisualizationController( BaseUIController, SharableMixin, UsesAnnotations,
else:
# Loading new visualization.
dataset = self.get_hda_or_ldda( trans, hda_ldda, dataset_id )
print 'dataset:', dataset
job = get_dataset_job( dataset )
viz_config = {
'dataset_id': dataset_id,
lib/galaxy/webapps/galaxy/controllers/workflow.py
+23 -57
View File
@@ -24,8 +24,8 @@ from galaxy.model.mapping import desc
from galaxy.tools.parameters import RuntimeValue, visit_input_values
from galaxy.tools.parameters.basic import DataToolParameter, DrillDownSelectToolParameter, SelectToolParameter, UnvalidatedValue
from galaxy.tools.parameters.grouping import Conditional, Repeat
from galaxy.util.json import from_json_string
from galaxy.util.odict import odict
from galaxy.util.json import to_json_string
from galaxy.util.sanitize_html import sanitize_html
from galaxy.util.topsort import CycleError, topsort, topsort_levels
from galaxy.web import error, url_for
@@ -81,7 +81,7 @@ class StoredWorkflowAllPublishedGrid( grids.Grid ):
default_filter = dict( public_url="All", username="All", tags="All" )
use_async = True
columns = [
grids.PublicURLColumn( "Name", key="name", filterable="advanced" ),
grids.PublicURLColumn( "Name", key="name", filterable="advanced", attach_popup=True ),
grids.OwnerAnnotationColumn( "Annotation", key="annotation", model_annotation_association_class=model.StoredWorkflowAnnotationAssociation, filterable="advanced" ),
grids.OwnerColumn( "Owner", key="username", model_class=model.User, filterable="advanced" ),
grids.CommunityRatingColumn( "Community Rating", key="rating" ),
@@ -94,7 +94,16 @@ class StoredWorkflowAllPublishedGrid( grids.Grid ):
cols_to_filter=[ columns[0], columns[1], columns[2], columns[4] ],
key="free-text-search", visible=False, filterable="standard" )
)
operations = []
operations = [
grids.GridOperation( "Import",
condition=( lambda item: not item.deleted ),
allow_multiple=False,
url_args=dict( action="imp" ) ),
grids.GridOperation( "Save as File",
condition=( lambda item: not item.deleted ),
allow_multiple=False,
url_args=dict( action="export_to_file" ) ),
]
def build_initial_query( self, trans, **kwargs ):
# Join so that searching stored_workflow.user makes sense.
@@ -132,7 +141,6 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
@web.expose
def index( self, trans ):
print trans.request.host_url
return self.list( trans )
@web.expose
@@ -589,8 +597,9 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
"""
# Load workflow from database
stored = self.get_stored_workflow( trans, id )
# Marke as deleted and save
# Mark as deleted and save
stored.deleted = True
trans.user.stored_workflow_menu_entries = [entry for entry in trans.user.stored_workflow_menu_entries if entry.stored_workflow != stored]
trans.sa_session.add( stored )
trans.sa_session.flush()
# Display the management page
@@ -890,56 +899,6 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
stored = self.get_stored_workflow( trans, id, check_ownership=False, check_accessible=True )
return trans.fill_template( "/workflow/export.mako", item=stored, use_panels=True )
@web.expose
@web.require_login( "use workflows" )
def import_from_myexp( self, trans, myexp_id, **kwd ):
"""
Imports a workflow from the myExperiment website.
Authenticates either by using HTTP basic, or with a cookie.
!!! This isn't currently used. MyExperiment uses the generic import_workflow method !!!
"""
# Parse parameters
myexp_username = kwd.get( 'myexp_username', None )
myexp_password = kwd.get( 'myexp_password', None )
myexp_cookie = kwd.get( 'myexp_cookie', None )
# Get workflow content.
conn = httplib.HTTPConnection( trans.app.config.get( "myexperiment_url", self.__myexp_url ) )
# NOTE: blocks web thread.
headers = {}
if myexp_username and myexp_password:
auth_header = base64.b64encode( '%s:%s' % ( myexp_username, myexp_password ))
headers = { "Authorization" : "Basic %s" % auth_header }
elif myexp_cookie:
headers = { "Cookie" : "myexperiment_session=%s" % myexp_cookie }
conn.request( "GET", "/workflow.xml?id=%s&elements=content" % myexp_id, headers=headers )
response = conn.getresponse()
response_data = response.read()
conn.close()
if response.status == 200:
parser = SingleTagContentsParser( "content" )
parser.feed( response_data )
workflow_content = base64.b64decode( parser.tag_content )
# Process workflow JSON and create workflow.
workflow_dict = from_json_string( workflow_content )
# Create workflow.
workflow, missing_tool_tups = self._workflow_from_dict( trans, workflow_dict, source="myExperiment" )
workflow = workflow.latest_workflow
if missing_tool_tups:
# TODO: handle the case where the imported workflow requires tools that are not available in
# the local Galaxy instance.
pass
# Provide user feedback.
workflow_list_str = " <br>Return to <a href='%s'>workflow list." % url_for( controller='workflow', action='list' )
if response.status != 200:
return trans.show_error_message( "There was a problem importing the workflow. Error: %s %s" % (response_data, workflow_list_str) )
if workflow.has_errors:
return trans.show_warn_message( "Imported, but some steps in this workflow have validation errors. %s" % workflow_list_str )
if workflow.has_cycles:
return trans.show_warn_message( "Imported, but this workflow contains cycles. %s" % workflow_list_str )
else:
return trans.show_message( "Workflow '%s' imported. %s" % (workflow.name, workflow_list_str) )
@web.expose
@web.require_login( "use workflows" )
def export_to_myexp( self, trans, id, myexp_username, myexp_password ):
@@ -1342,6 +1301,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
# Build the state for each step
errors = {}
has_upgrade_messages = False
step_version_changes = []
# has_errors is never used
# has_errors = False
saved_history = None
@@ -1499,6 +1459,8 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
step.upgrade_messages = step.module.check_and_update_state()
if step.upgrade_messages:
has_upgrade_messages = True
if step.type == 'tool' and step.module.version_changes:
step_version_changes.extend(step.module.version_changes)
# Any connected input needs to have value DummyDataset (these
# are not persisted so we need to do it every time)
step.module.add_dummy_datasets( connections=step.input_connections )
@@ -1529,6 +1491,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
steps=workflow.steps,
workflow=stored,
has_upgrade_messages=has_upgrade_messages,
step_version_changes=step_version_changes,
errors=errors,
incoming=kwargs,
history_id=history_id,
@@ -1822,12 +1785,15 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
# FIXME: Position should be handled inside module
step.position = step_dict['position']
module = module_factory.from_dict( trans, step_dict, secure=False )
module.save_to_step( step )
if module.type == 'tool' and module.tool is None:
# A required tool is not available in the local Galaxy instance.
missing_tool_tup = ( step_dict[ 'tool_id' ], step_dict[ 'name' ], step_dict[ 'tool_version' ] )
if missing_tool_tup not in missing_tool_tups:
missing_tool_tups.append( missing_tool_tup )
module.save_to_step( step )
# Save the entire step_dict in the unused config field, be parsed later
# when we do have the too when we do have the tool
step.config = to_json_string(step_dict)
if step.tool_errors:
workflow.has_errors = True
# Stick this in the step temporarily
@@ -1870,7 +1836,7 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
stored.user = trans.user
if data[ 'annotation' ]:
self.add_item_annotation( trans.sa_session, stored.user, stored, data[ 'annotation' ] )
# Persist
trans.sa_session.add( stored )
trans.sa_session.flush()
lib/galaxy/webapps/reports/controllers/system.py
+3 -3
View File
@@ -1,9 +1,9 @@
import operator, os
import os
import logging
from datetime import datetime, timedelta
from galaxy.web.base.controller import BaseUIController, web
from galaxy import model, util
from galaxy.model.orm import and_, not_, or_, desc
import logging
from galaxy.model.orm import and_, desc, eagerload
log = logging.getLogger( __name__ )
class System( BaseUIController ):
lib/galaxy/webapps/tool_shed/app.py
+3
View File
@@ -7,6 +7,7 @@ import galaxy.webapps.tool_shed.model
from galaxy.openid.providers import OpenIDProviders
from galaxy.web import security
from galaxy.tags.tag_handler import CommunityTagHandler
import tool_shed.repository_types.registry
class UniverseApplication( object ):
"""Encapsulates the state of a Universe application"""
@@ -20,6 +21,8 @@ class UniverseApplication( object ):
# Set up datatypes registry
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
# Set up the repository_types registry.
self.repository_types_registry = tool_shed.repository_types.registry.Registry()
# Determine the database url
if self.config.database_connection:
db_url = self.config.database_connection
lib/galaxy/webapps/tool_shed/config.py
+4 -1
View File
@@ -6,7 +6,7 @@ import sys, os
import logging, logging.config
from optparse import OptionParser
import ConfigParser
from galaxy.util import string_as_bool
from galaxy.util import string_as_bool, listify
from galaxy import eggs
import pkg_resources
@@ -44,6 +44,9 @@ class Configuration( object ):
self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
# Tool stuff
self.tool_filters = listify( kwargs.get( "tool_filters", [] ) )
self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ) )
self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ) )
self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
self.tool_secret = kwargs.get( "tool_secret", "" )
self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "shed-tool-data" ), os.getcwd() )
lib/galaxy/webapps/tool_shed/controllers/admin.py
+27 -36
View File
@@ -31,7 +31,7 @@ class AdminController( BaseUIController, Admin ):
# with an "f-" prefix, simulating filtering by clicking a search link. We have
# to take this approach because the "-" character is illegal in HTTP requests.
if 'operation' in kwd:
operation = kwd['operation'].lower()
operation = kwd[ 'operation' ].lower()
if operation == "view_or_manage_repository":
return trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repositories',
@@ -119,12 +119,11 @@ class AdminController( BaseUIController, Admin ):
@web.expose
@web.require_admin
def create_category( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
name = util.restore_text( params.get( 'name', '' ) ).strip()
description = util.restore_text( params.get( 'description', '' ) ).strip()
if params.get( 'create_category_button', False ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
name = kwd.get( 'name', '' ).strip()
description = kwd.get( 'description', '' ).strip()
if kwd.get( 'create_category_button', False ):
if not name or not description:
message = 'Enter a valid name and a description'
status = 'error'
@@ -151,9 +150,8 @@ class AdminController( BaseUIController, Admin ):
@web.expose
@web.require_admin
def delete_repository( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if id:
# Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
@@ -188,9 +186,8 @@ class AdminController( BaseUIController, Admin ):
@web.expose
@web.require_admin
def delete_repository_metadata( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if id:
ids = util.listify( id )
@@ -213,10 +210,9 @@ class AdminController( BaseUIController, Admin ):
@web.expose
@web.require_admin
def edit_category( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
id = params.get( 'id', None )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if not id:
message = "No category ids received for editing"
trans.response.send_redirect( web.url_for( controller='admin',
@@ -224,14 +220,14 @@ class AdminController( BaseUIController, Admin ):
message=message,
status='error' ) )
category = suc.get_category( trans, id )
if params.get( 'edit_category_button', False ):
new_name = util.restore_text( params.get( 'name', '' ) ).strip()
new_description = util.restore_text( params.get( 'description', '' ) ).strip()
if kwd.get( 'edit_category_button', False ):
new_name = kwd.get( 'name', '' ).strip()
new_description = kwd.get( 'description', '' ).strip()
if category.name != new_name or category.description != new_description:
if not new_name:
message = 'Enter a valid name'
status = 'error'
elif category.name != new_name and suc.get_category_by_name( trans, name ):
elif category.name != new_name and suc.get_category_by_name( trans, new_name ):
message = 'A category with that name already exists'
status = 'error'
else:
@@ -288,9 +284,8 @@ class AdminController( BaseUIController, Admin ):
@web.expose
@web.require_admin
def regenerate_statistics( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
if 'regenerate_statistics_button' in kwd:
trans.app.shed_counter.generate_statistics()
message = "Successfully regenerated statistics"
@@ -315,9 +310,8 @@ class AdminController( BaseUIController, Admin ):
@web.expose
@web.require_admin
def undelete_repository( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if id:
# Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
@@ -358,9 +352,8 @@ class AdminController( BaseUIController, Admin ):
# TODO: We should probably eliminate the Category.deleted column since it really makes no
# sense to mark a category as deleted (category names and descriptions can be changed instead).
# If we do this, and the following 2 methods can be eliminated.
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if id:
ids = util.listify( id )
@@ -385,9 +378,8 @@ class AdminController( BaseUIController, Admin ):
# This method should only be called for a Category that has previously been deleted.
# Purging a deleted Category deletes all of the following from the database:
# - RepoitoryCategoryAssociations where category_id == Category.id
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if id:
ids = util.listify( id )
@@ -414,9 +406,8 @@ class AdminController( BaseUIController, Admin ):
@web.expose
@web.require_admin
def undelete_category( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if id:
ids = util.listify( id )
lib/galaxy/webapps/tool_shed/controllers/repository.py
+180 -424
View File
File diff suppressed because it is too large Load Diff
lib/galaxy/webapps/tool_shed/controllers/repository_review.py
+30 -42
View File
@@ -34,9 +34,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.require_login( "approve repository review" )
def approve_repository_review( self, trans, **kwd ):
# The value of the received id is the encoded review id.
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
encoded_review_id = kwd[ 'id' ]
review = review_util.get_review( trans, encoded_review_id )
if kwd.get( 'approve_repository_review_button', False ):
@@ -69,9 +68,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.expose
@web.require_login( "browse review" )
def browse_review( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
review = review_util.get_review( trans, kwd[ 'id' ] )
repository = review.repository
repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) )
@@ -101,12 +99,11 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.expose
@web.require_login( "create component" )
def create_component( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
name = util.restore_text( params.get( 'name', '' ) )
description = util.restore_text( params.get( 'description', '' ) )
if params.get( 'create_component_button', False ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
name = kwd.get( 'name', '' )
description = kwd.get( 'description', '' )
if kwd.get( 'create_component_button', False ):
if not name or not description:
message = 'Enter a valid name and a description'
status = 'error'
@@ -133,9 +130,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.require_login( "create review" )
def create_review( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
changeset_revision = kwd.get( 'changeset_revision', None )
previous_review_id = kwd.get( 'previous_review_id', None )
@@ -197,10 +193,9 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.expose
@web.require_login( "edit component" )
def edit_component( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
id = params.get( 'id', None )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if not id:
message = "No component ids received for editing"
trans.response.send_redirect( web.url_for( controller='admin',
@@ -208,8 +203,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
message=message,
status='error' ) )
component = review_util.get_component( trans, id )
if params.get( 'edit_component_button', False ):
new_description = util.restore_text( params.get( 'description', '' ) ).strip()
if kwd.get( 'edit_component_button', False ):
new_description = kwd.get( 'description', '' ).strip()
if component.description != new_description:
component.description = new_description
trans.sa_session.add( component )
@@ -229,9 +224,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.require_login( "edit review" )
def edit_review( self, trans, **kwd ):
# The value of the received id is the encoded review id.
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
review_id = kwd.get( 'id', None )
review = review_util.get_review( trans, review_id )
components_dict = odict()
@@ -403,9 +397,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.require_login( "manage repositories reviewed by me" )
def manage_repositories_reviewed_by_me( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
if 'operation' in kwd:
kwd[ 'mine' ] = True
return trans.response.send_redirect( web.url_for( controller='repository_review',
@@ -471,9 +464,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.require_login( "manage repository reviews" )
def manage_repository_reviews( self, trans, mine=False, **kwd ):
# The value of the received id is the encoded repository id.
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
if repository_id:
repository = suc.get_repository_in_tool_shed( trans, repository_id )
@@ -519,9 +511,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.require_login( "manage repository reviews of revision" )
def manage_repository_reviews_of_revision( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
changeset_revision = kwd.get( 'changeset_revision', None )
repository = suc.get_repository_in_tool_shed( trans, repository_id )
@@ -542,9 +533,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.expose
@web.require_login( "repository reviews by user" )
def repository_reviews_by_user( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
if 'operation' in kwd:
operation = kwd['operation'].lower()
@@ -569,9 +559,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.expose
@web.require_login( "reviewed repositories i own" )
def reviewed_repositories_i_own( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
# The value of the received id is the encoded repository id.
if 'operation' in kwd:
operation = kwd['operation'].lower()
@@ -589,9 +578,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ):
@web.require_login( "select previous review" )
def select_previous_review( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository = suc.get_repository_in_tool_shed( trans, kwd[ 'id' ] )
changeset_revision = kwd.get( 'changeset_revision', None )
repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) )

Some files were not shown because too many files have changed in this diff Show More