From ca8cc1420ef80bd2fef17fa9bb21dbb0714b4a88 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 3 Aug 2007 16:37:44 +0000 Subject: [PATCH] Add the ability for secondary (child) and new primary datasets to be collected automatically. Datasets can now be set to not be visible, which will prevent their display in a user's history. To have a child dataset harvested automatically, simply name the file: child_PARENTID_DESIGNATION_VISIBILITY_EXT (To make a new primary dataset, simply use the form: primary_ASSOCIATEDWITHDATASETID_DESIGNATION_VISIBILITY_EXT) and place this file in the directory specified by $__new_file_path__ For example: You define the command in the tool XML as: some_command.py $input1 $out_file1 $out_file1.id $__new_file_path__ Suppose the input dataset is 1 and the output dataset is 2, the commandline becomes: python2.4 some_command.py ./database/files/dataset_1.dat ./database/files/dataset_2.dat 2 ./database/tmp In addition to the primary file (a HTML file), this program creates files in the ./database/tmp directory named: child_2_SomeImage_invisible_jpg child_2_SomeText_visible_text These files are discovered and added as children to the appropriate dataset. The text file will appear in the user's history, but the jpg will not. All files can be viewed however, by making links in the primary (HTML) history item like: Some Image Some Text Designations need to be unique for each originally declared output file, simply using a counter can work well. These new files can be accessed in the exec_after_process hook under param_dict['__collected_datasets__']. Note the update to the universe_wsgi.ini.sample file. Some database changes are required: ALTER TABLE dataset ADD visible boolean; UPDATE dataset SET visible = true WHERE visible is NULL; --- lib/galaxy/config.py | 1 + lib/galaxy/jobs/__init__.py | 4 ++ lib/galaxy/model/__init__.py | 7 +-- lib/galaxy/model/mapping.py | 1 + lib/galaxy/tools/__init__.py | 85 +++++++++++++++++++++++++++++- lib/galaxy/web/controllers/root.py | 15 ++++++ templates/history.tmpl | 35 +++++++----- universe_wsgi.ini.sample | 2 + 8 files changed, 132 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index bbb817e8a67..e8b598fe639 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -25,6 +25,7 @@ class Configuration( object ): self.database = resolve_path( kwargs.get( "database_file", "database/universe.d" ), self.root ) self.database_connection = kwargs.get( "database_connection", False ) self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root ) + self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root ) self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root ) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 68582a8d4b7..deca35fd6a2 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -286,6 +286,10 @@ class JobWrapper( object ): out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) param_dict = self.tool.params_from_strings( param_dict, self.app ) + # Create generated output children and primary datasets and add to param_dict + collected_datasets = {'children':self.tool.collect_child_datasets(out_data),'primary':self.tool.collect_primary_datasets(out_data)} + param_dict.update({'__collected_datasets__':collected_datasets}) + # Call 'exec_after_process' hook self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data, out_data=out_data, param_dict=param_dict, tool=self.tool, stdout=stdout, stderr=stderr ) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 96ef9d6f873..db0b7f59644 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -143,7 +143,7 @@ class Dataset( object ): engine = None def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, dbkey=None, state=None, metadata=None, history=None, parent_id=None, designation=None, - validation_errors=None ): + validation_errors=None, visible=True ): self.name = name or "Unnamed dataset" self.id = id self.hid = hid @@ -158,6 +158,7 @@ class Dataset( object ): self.designation = designation self.deleted = False self.purged = False + self.visible = visible # Relationships self.history = history self.validation_errors = validation_errors @@ -267,9 +268,9 @@ class Dataset( object ): # if data.parent_id and data.parent_id == self.id: # if designation == data.designation: # return data - for child_assocation in self.children: + for child_association in self.children: if child_association.designation == designation: - return child + return child_association.child return None def purge( self ): """Removes the file contents from disk """ diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index d38bc1e0597..b8b9c43e0c4 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -73,6 +73,7 @@ Dataset.table = Table( "dataset", metadata, Column( "designation", TrimmedString( 255 ) ), Column( "deleted", Boolean ), Column( "purged", Boolean ), + Column( "visible", Boolean ), ForeignKeyConstraint(['parent_id'],['dataset.id'], ondelete="CASCADE") ) ValidationError.table = Table( "validation_error", metadata, diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 99e2492d6ce..c2239c4bd7c 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -6,7 +6,7 @@ import pkg_resources; pkg_resources.require( "Cheetah" ) pkg_resources.require( "simplejson" ) -import logging, os, string, sys, tempfile +import logging, os, string, sys, tempfile, glob, shutil import simplejson import sha, hmac, binascii @@ -782,7 +782,10 @@ class Tool: child = child_association.child key = "_CHILD___%s___%s" % ( name, child.designation ) param_dict[ key ] = DatasetFilenameWrapper( child ) - # Return the dictionary of parameters + # We add access to app here, this allows access to app.config, etc + param_dict['__app__'] = RawObjectWrapper( self.app ) + param_dict['__new_file_path__'] = self.app.config.new_file_path #More convienent access to this value; we don't need to wrap a string + # Return the dictionary of parameters return param_dict def build_param_file( self, param_dict, directory=None ): @@ -859,12 +862,90 @@ class Tool: e.args = ( "Error in '%s' hook '%s', original message: %s" % ( self.name, hook_name, e.args[0] ) ) raise + def collect_child_datasets( self, output): + children = {} + #Loop through output file names, looking for generated children in form of 'child_parentId_designation_visibility_extension' + for name, outdata in output.items(): + for filename in glob.glob(os.path.join(self.app.config.new_file_path,"child_%i_*" % outdata.id) ): + if not name in children: + children[name] = {} + fields = os.path.basename(filename).split("_") + fields.pop(0) + parent_id = int(fields.pop(0)) + designation = fields.pop(0) + visible = fields.pop(0).lower() + if visible == "visible": visible = True + else: visible = False + ext = fields.pop(0).lower() + # Create new child dataset + child_data = self.app.model.Dataset(extension=ext, parent_id=parent_id, designation=designation, visible=visible, dbkey=outdata.dbkey) + child_data.flush() + # Move data from temp location to dataset location + shutil.move(filename, child_data.file_name) + child_data.name = "Secondary Dataset (%s)" % (designation) + child_data.state = child_data.states.OK + child_data.init_meta() + child_data.set_peek() + child_data.flush() + # Add to child accociation table + assoc = self.app.model.DatasetChildAssociation() + assoc.child = child_data + assoc.designation = child_data.designation + outdata.children.append( assoc ) + # Add child to return dict + children[name][designation] = child_data + return children + + def collect_primary_datasets( self, output): + primary_datasets = {} + #Loop through output file names, looking for generated primary datasets in form of 'primary_associatedWithDatasetID_designation_visibility_extension' + for name, outdata in output.items(): + for filename in glob.glob(os.path.join(self.app.config.new_file_path,"primary_%i_*" % outdata.id) ): + if not name in primary_datasets: + primary_datasets[name] = {} + fields = os.path.basename(filename).split("_") + fields.pop(0) + parent_id = int(fields.pop(0)) + designation = fields.pop(0) + visible = fields.pop(0).lower() + if visible == "visible": visible = True + else: visible = False + ext = fields.pop(0).lower() + # Create new primary dataset + primary_data = self.app.model.Dataset(extension=ext, designation=designation, visible=visible, dbkey=outdata.dbkey) + primary_data.flush() + self.app.model.History.get(outdata.history_id).add_dataset(primary_data) + # Move data from temp location to dataset location + shutil.move(filename, primary_data.file_name) + primary_data.name = outdata.name + primary_data.info = outdata.info + primary_data.state = primary_data.states.OK + primary_data.init_meta(copy_from=outdata) + primary_data.set_peek() + primary_data.flush() + # Add dataset to return dict + primary_datasets[name][designation] = primary_data + return primary_datasets + + # ---- Utility classes to be factored out ----------------------------------- class BadValue( object ): def __init__( self, value ): self.value = value + +class RawObjectWrapper( object ): + """ + Wraps an object so that __str__ returns module_name:class_name. + """ + def __init__( self, obj ): + self.obj = obj + def __str__( self ): + return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + def __getattr__( self, key ): + return getattr( self.obj, key ) + class InputValueWrapper( object ): """ Wraps an input so that __str__ gives the "param_dict" representation. diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index e9d10606098..ca29fd0574a 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -122,6 +122,21 @@ class Universe( BaseController ): else: return "No data with id=%d" % id + @web.expose + def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"): + """ + Returns child data directly into the browser, based upon parent_id and designation. + """ + try: + data = self.app.model.Dataset.get( parent_id ) + if data: + child = data.get_child_by_designation(designation) + if child: + return self.display(trans, id=child.id, tofile=tofile, toext=toext) + except Exception: + pass + return "A child named %s could not be found for data %s" % ( designation, parent_id ) + @web.expose def display_as( self, trans, id=None, display_app=None, **kwd ): """Returns a file in a format that can successfully be displayed in display_app""" diff --git a/templates/history.tmpl b/templates/history.tmpl index 44ab9e72eea..d795c5d3fcf 100644 --- a/templates/history.tmpl +++ b/templates/history.tmpl @@ -257,7 +257,7 @@ div#footer {
#if $data.has_data: save - #for $display_app in $data.datatype.get_display_types(): + #for $display_app in $data.datatype.get_display_types(): #set $display_links = $data.datatype.get_display_links($data, $display_app, $app, $request.base) #if $len($display_links) > 0: | $data.datatype.get_display_label($display_app) @@ -276,18 +276,25 @@ div#footer { ## ## Child datasets ## - + #if $len( $data.children ) > 0: -
- There are ${len( $data.children )} secondary datasets. - #for $idx, $child_assoc in $enumerate($data.children) - #set $child = $child_assoc.child - $render_dataset( $child, $idx + 1 ) - #end for -
- #end if - -
+ #set $children = [] + #for $child_assoc in $data.children: + #if $child_assoc.child.visible: + $children.append($child_assoc.child) + #end if + #end for + #if $len( $children ) > 0: +
+ There are ${len( $children )} secondary datasets. + #for $idx, $child in $enumerate($children) + $render_dataset( $child, $idx + 1 ) + #end for +
+ #end if + #end if + + #end def @@ -299,7 +306,9 @@ div#footer { #else ## Render all active (not deleted) datasets, ordered from newest to oldest #for $data in reversed( $history.active_datasets ) - $render_dataset( $data, $data.hid ) + #if $data.visible: + $render_dataset( $data, $data.hid ) + #end if #end for #end if diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index b246b2bee7a..82a52eed4da 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -29,6 +29,8 @@ database_file = database/universe.sqlite # Where dataset files are saved file_path = database/files +# Temporary storage for additional datasets, this should be shared through the cluster +new_file_path = database/tmp # Tools tool_config_file = tool_conf.xml