opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add the ability for secondary (child) and new primary datasets to be collected automatically. Datasets can now be set to not be visible, which will prevent their display in a user's history.

Browse Source 
To have a child dataset harvested automatically, simply name the file:
child_PARENTID_DESIGNATION_VISIBILITY_EXT (To make a new primary dataset, simply use the form: primary_ASSOCIATEDWITHDATASETID_DESIGNATION_VISIBILITY_EXT)
and place this file in the directory specified by $__new_file_path__

For example:
You define the command in the tool XML as:
<command interpreter="python2.4">some_command.py $input1 $out_file1 $out_file1.id $__new_file_path__</command>

Suppose the input dataset is 1 and the output dataset is 2, the commandline becomes:
python2.4 some_command.py ./database/files/dataset_1.dat ./database/files/dataset_2.dat 2 ./database/tmp

In addition to the primary file (a HTML file), this program creates files in the ./database/tmp directory named:
child_2_SomeImage_invisible_jpg
child_2_SomeText_visible_text

These files are discovered and added as children to the appropriate dataset. The text file will appear in the user's history, but the jpg will not.

All files can be viewed however, by making links in the primary (HTML) history item like:
<img src="display_child?parent_id=2&designation=SomeImage" alt="Some Image"/>
<a href="display_child?parent_id=2&designation=SomeText">Some Text</a>


Designations need to be unique for each originally declared output file, simply using a counter can work well.

These new files can be accessed in the exec_after_process hook under param_dict['__collected_datasets__'].

Note the update to the universe_wsgi.ini.sample file.

Some database changes are required:
ALTER TABLE dataset ADD visible boolean;
UPDATE dataset SET visible = true WHERE visible is NULL;
This commit is contained in:
Daniel Blankenberg
2007-08-03 16:37:44 +00:00
parent 868c5a26b2
commit ca8cc1420e
8 changed files with 132 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/config.py
+1
View File
@@ -25,6 +25,7 @@ class Configuration( object ):
self.database = resolve_path( kwargs.get( "database_file", "database/universe.d" ), self.root )
self.database_connection = kwargs.get( "database_connection", False )
self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root )
self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root )
lib/galaxy/jobs/__init__.py
+4
View File
@@ -286,6 +286,10 @@ class JobWrapper( object ):
out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] )
param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )
param_dict = self.tool.params_from_strings( param_dict, self.app )
# Create generated output children and primary datasets and add to param_dict
collected_datasets = {'children':self.tool.collect_child_datasets(out_data),'primary':self.tool.collect_primary_datasets(out_data)}
param_dict.update({'__collected_datasets__':collected_datasets})
# Call 'exec_after_process' hook
self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data,
out_data=out_data, param_dict=param_dict,
tool=self.tool, stdout=stdout, stderr=stderr )
lib/galaxy/model/__init__.py
+4 -3
View File
@@ -143,7 +143,7 @@ class Dataset( object ):
engine = None
def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None,
dbkey=None, state=None, metadata=None, history=None, parent_id=None, designation=None,
validation_errors=None ):
validation_errors=None, visible=True ):
self.name = name or "Unnamed dataset"
self.id = id
self.hid = hid
@@ -158,6 +158,7 @@ class Dataset( object ):
self.designation = designation
self.deleted = False
self.purged = False
self.visible = visible
# Relationships
self.history = history
self.validation_errors = validation_errors
@@ -267,9 +268,9 @@ class Dataset( object ):
# if data.parent_id and data.parent_id == self.id:
# if designation == data.designation:
# return data
for child_assocation in self.children:
for child_association in self.children:
if child_association.designation == designation:
return child
return child_association.child
return None
def purge( self ):
"""Removes the file contents from disk """
lib/galaxy/model/mapping.py
+1
View File
@@ -73,6 +73,7 @@ Dataset.table = Table( "dataset", metadata,
Column( "designation", TrimmedString( 255 ) ),
Column( "deleted", Boolean ),
Column( "purged", Boolean ),
Column( "visible", Boolean ),
ForeignKeyConstraint(['parent_id'],['dataset.id'], ondelete="CASCADE") )
ValidationError.table = Table( "validation_error", metadata,
lib/galaxy/tools/__init__.py
+83 -2
View File
@@ -6,7 +6,7 @@ import pkg_resources;
pkg_resources.require( "Cheetah" )
pkg_resources.require( "simplejson" )
import logging, os, string, sys, tempfile
import logging, os, string, sys, tempfile, glob, shutil
import simplejson
import sha, hmac, binascii
@@ -782,7 +782,10 @@ class Tool:
child = child_association.child
key = "_CHILD___%s___%s" % ( name, child.designation )
param_dict[ key ] = DatasetFilenameWrapper( child )
# Return the dictionary of parameters
# We add access to app here, this allows access to app.config, etc
param_dict['__app__'] = RawObjectWrapper( self.app )
param_dict['__new_file_path__'] = self.app.config.new_file_path #More convienent access to this value; we don't need to wrap a string
# Return the dictionary of parameters
return param_dict
def build_param_file( self, param_dict, directory=None ):
@@ -859,12 +862,90 @@ class Tool:
e.args = ( "Error in '%s' hook '%s', original message: %s" % ( self.name, hook_name, e.args[0] ) )
raise
def collect_child_datasets( self, output):
children = {}
#Loop through output file names, looking for generated children in form of 'child_parentId_designation_visibility_extension'
for name, outdata in output.items():
for filename in glob.glob(os.path.join(self.app.config.new_file_path,"child_%i_*" % outdata.id) ):
if not name in children:
children[name] = {}
fields = os.path.basename(filename).split("_")
fields.pop(0)
parent_id = int(fields.pop(0))
designation = fields.pop(0)
visible = fields.pop(0).lower()
if visible == "visible": visible = True
else: visible = False
ext = fields.pop(0).lower()
# Create new child dataset
child_data = self.app.model.Dataset(extension=ext, parent_id=parent_id, designation=designation, visible=visible, dbkey=outdata.dbkey)
child_data.flush()
# Move data from temp location to dataset location
shutil.move(filename, child_data.file_name)
child_data.name = "Secondary Dataset (%s)" % (designation)
child_data.state = child_data.states.OK
child_data.init_meta()
child_data.set_peek()
child_data.flush()
# Add to child accociation table
assoc = self.app.model.DatasetChildAssociation()
assoc.child = child_data
assoc.designation = child_data.designation
outdata.children.append( assoc )
# Add child to return dict
children[name][designation] = child_data
return children
def collect_primary_datasets( self, output):
primary_datasets = {}
#Loop through output file names, looking for generated primary datasets in form of 'primary_associatedWithDatasetID_designation_visibility_extension'
for name, outdata in output.items():
for filename in glob.glob(os.path.join(self.app.config.new_file_path,"primary_%i_*" % outdata.id) ):
if not name in primary_datasets:
primary_datasets[name] = {}
fields = os.path.basename(filename).split("_")
fields.pop(0)
parent_id = int(fields.pop(0))
designation = fields.pop(0)
visible = fields.pop(0).lower()
if visible == "visible": visible = True
else: visible = False
ext = fields.pop(0).lower()
# Create new primary dataset
primary_data = self.app.model.Dataset(extension=ext, designation=designation, visible=visible, dbkey=outdata.dbkey)
primary_data.flush()
self.app.model.History.get(outdata.history_id).add_dataset(primary_data)
# Move data from temp location to dataset location
shutil.move(filename, primary_data.file_name)
primary_data.name = outdata.name
primary_data.info = outdata.info
primary_data.state = primary_data.states.OK
primary_data.init_meta(copy_from=outdata)
primary_data.set_peek()
primary_data.flush()
# Add dataset to return dict
primary_datasets[name][designation] = primary_data
return primary_datasets
# ---- Utility classes to be factored out -----------------------------------
class BadValue( object ):
def __init__( self, value ):
self.value = value
class RawObjectWrapper( object ):
"""
Wraps an object so that __str__ returns module_name:class_name.
"""
def __init__( self, obj ):
self.obj = obj
def __str__( self ):
return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__)
def __getattr__( self, key ):
return getattr( self.obj, key )
class InputValueWrapper( object ):
"""
Wraps an input so that __str__ gives the "param_dict" representation.
lib/galaxy/web/controllers/root.py
+15
View File
@@ -122,6 +122,21 @@ class Universe( BaseController ):
else:
return "No data with id=%d" % id
@web.expose
def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"):
"""
Returns child data directly into the browser, based upon parent_id and designation.
"""
try:
data = self.app.model.Dataset.get( parent_id )
if data:
child = data.get_child_by_designation(designation)
if child:
return self.display(trans, id=child.id, tofile=tofile, toext=toext)
except Exception:
pass
return "A child named %s could not be found for data %s" % ( designation, parent_id )
@web.expose
def display_as( self, trans, id=None, display_app=None, **kwd ):
"""Returns a file in a format that can successfully be displayed in display_app"""
templates/history.tmpl
+22 -13
View File
@@ -257,7 +257,7 @@ div#footer {
<div>
#if $data.has_data:
<a href="display?id=$data.id&tofile=yes&toext=$data.ext" target="_blank">save</a>
#for $display_app in $data.datatype.get_display_types():
#for $display_app in $data.datatype.get_display_types():
#set $display_links = $data.datatype.get_display_links($data, $display_app, $app, $request.base)
#if $len($display_links) > 0:
| $data.datatype.get_display_label($display_app)
@@ -276,18 +276,25 @@ div#footer {
##
## Child datasets
##
#if $len( $data.children ) > 0:
<div>
There are ${len( $data.children )} secondary datasets.
#for $idx, $child_assoc in $enumerate($data.children)
#set $child = $child_assoc.child
$render_dataset( $child, $idx + 1 )
#end for
</div>
#end if
</div>
#set $children = []
#for $child_assoc in $data.children:
#if $child_assoc.child.visible:
$children.append($child_assoc.child)
#end if
#end for
#if $len( $children ) > 0:
<div>
There are ${len( $children )} secondary datasets.
#for $idx, $child in $enumerate($children)
$render_dataset( $child, $idx + 1 )
#end for
</div>
#end if
#end if
</div>
</div>
#end def
@@ -299,7 +306,9 @@ div#footer {
#else
## Render all active (not deleted) datasets, ordered from newest to oldest
#for $data in reversed( $history.active_datasets )
$render_dataset( $data, $data.hid )
#if $data.visible:
$render_dataset( $data, $data.hid )
#end if
#end for
#end if
universe_wsgi.ini.sample
+2
View File
@@ -29,6 +29,8 @@ database_file = database/universe.sqlite
# Where dataset files are saved
file_path = database/files
# Temporary storage for additional datasets, this should be shared through the cluster
new_file_path = database/tmp
# Tools
tool_config_file = tool_conf.xml