From fda163bbe8e5438b1f1446a99156637b158dd024 Mon Sep 17 00:00:00 2001 From: Ian Schenck Date: Mon, 12 Mar 2007 18:35:03 +0000 Subject: [PATCH] Data validation. Please please please take a look and give me some input. --- lib/galaxy/datatypes/data.py | 3 ++ lib/galaxy/datatypes/interval.py | 4 ++ lib/galaxy/interfaces/root.py | 27 ++++++++++- lib/galaxy/tools/__init__.py | 15 ++++-- lib/galaxy/tools/actions/upload.py | 4 +- templates/history.tmpl | 10 ++-- templates/tool_menu.tmpl | 2 + tool_conf.xml.sample | 1 + tools/validation/fix_errors.py | 58 +++++++++++++++++++++++ tools/validation/fix_errors.xml | 22 +++++++++ tools/validation/fix_errors_code.py | 49 +++++++++++++++++++ {scripts => tools/validation}/validate.py | 0 12 files changed, 182 insertions(+), 13 deletions(-) create mode 100644 tools/validation/fix_errors.py create mode 100644 tools/validation/fix_errors.xml create mode 100644 tools/validation/fix_errors_code.py rename {scripts => tools/validation}/validate.py (100%) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 5827de8ed5a..8a9c5ba05db 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -39,6 +39,9 @@ class Data( object ): def validate(self, dataset): """Unimplemented validate, return no exceptions""" return list() + def repair_methods(self, dataset): + """Unimplemented method, returns dict with method/option for repairing errors""" + return None class Text( Data ): def write_from_stream(self, stream): diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 6d75d5efa83..3a631b87c97 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -186,6 +186,10 @@ class Interval( Tabular ): infile.close() return errors + def repair_methods( self, dataset ): + """Return options for removing errors along with a description""" + return [("lines","Remove erroneous lines")] + class Bed( Interval ): """Tab delimited data in BED format""" def init_meta( self, dataset ): diff --git a/lib/galaxy/interfaces/root.py b/lib/galaxy/interfaces/root.py index cb8a3edf55b..cd0cb3d4407 100644 --- a/lib/galaxy/interfaces/root.py +++ b/lib/galaxy/interfaces/root.py @@ -7,6 +7,8 @@ import re, socket from galaxy import util, datatypes, jobs, web, util import common from cgi import escape, FieldStorage +import urllib +import traceback log = logging.getLogger( __name__ ) @@ -496,6 +498,29 @@ class Universe(common.Root): def masthead( self, trans ): return trans.fill_template( "masthead.tmpl" ) + @web.expose + def dataset_errors( self, trans, id=None, **kwd ): + """View/fix errors associated with dataset""" + data = trans.app.model.Dataset.get( id ) + p = kwd + if p.get("fix_errors", None): + # launch tool to create new, (hopefully) error free dataset + tool_params = {} + tool_params["tool_id"] = 'fix_errors' + tool_params["runtool_btn"] = 'T' + tool_params["input"] = id + # send methods selected + repair_methods = data.datatype.repair_methods( data ) + methods = [] + for method, description in repair_methods: + if method in p: methods.append(method) + tool_params["methods"] = ",".join(methods) + url = "/tool_runner/index?" + urllib.urlencode(tool_params) + trans.response.send_redirect(url) + else: + history = trans.app.model.History.get( data.history_id ) + return trans.fill_template('dataset_errors.tmpl', data=data, history=history) + # ---- Debug methods ---------------------------------------------------- @web.expose @@ -551,4 +576,4 @@ class Universe(common.Root): new_child.flush() des.hid_counter = src.hid_counter self.app.model.flush() - return des \ No newline at end of file + return des diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index b7dc54adab4..37d4207d427 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -126,6 +126,9 @@ class Tool: self.command = '' # Short description of the tool self.description = util.xml_text(root, "description") + # Is this a 'hidden' tool (hidden in tool menu) + self.hidden = util.xml_text(root, "hidden") + if self.hidden: self.hidden = util.string_as_bool(self.hidden) # Load any tool specific code (optional) self.code_namespace = dict() for code_elem in root.findall("code"): @@ -620,16 +623,18 @@ class DefaultToolAction( object ): # Store data after custom code runs trans.app.model.flush() - - # Build the job's command line + + # Build params, done before hook so hook can use param_dict = tool.build_param_dict( incoming, inp_data, out_data ) - param_filename = tool.build_param_file( param_dict ) - command_line = tool.build_command_line( param_dict ) - + # Run the before queue ("exec_before_job") hook # FIXME: this hook should probably be called exec_before_job_queued tool.call_hook( 'exec_before_job', trans, inp_data=inp_data, out_data=out_data, tool=tool, param_dict=param_dict ) + + # Build the job's command line, moved to after the hook so the hook can alter params + param_filename = tool.build_param_file( param_dict ) + command_line = tool.build_command_line( param_dict ) # Create the job object job = trans.app.model.Job() diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index a62eeec71e4..fffc52d1bb2 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -1,7 +1,7 @@ import os, shutil, urllib, StringIO from galaxy import datatypes, jobs from galaxy.datatypes import sniff -from galaxy import model +from galaxy import model, util import sys, traceback @@ -89,7 +89,7 @@ class UploadToolAction( object ): # validate incomming data for error in data.datatype.validate( data ): data.add_validation_error( - model.ValidationError( message=str( error ), err_type=error.__class__.__name__, attributes=str( error.__dict__ ) ) ) + model.ValidationError( message=str( error ), err_type=error.__class__.__name__, attributes=util.object_to_string( error.__dict__ ) ) ) trans.history.add_dataset( data ) trans.app.model.flush() return data diff --git a/templates/history.tmpl b/templates/history.tmpl index 891fc8ebc2e..cc932d040a5 100644 --- a/templates/history.tmpl +++ b/templates/history.tmpl @@ -199,6 +199,11 @@ main();">
Info: $data.display_info
+ #if $len($data.validation_errors)>0 +
+ There are $len($data.validation_errors) possibles errors in this dataset (view). +
+ #end if
#if $data.ext in [ "bed", "interval", "tabular", "txt", "text", "axt", "maf", "fasta", "gff", "gmaj.zip" ]: save @@ -216,11 +221,6 @@ main();"> #if $data.peek != "no peek" 
$data.display_peek
#end if - #if $len($data.validation_errors)>0 -
- There are $len($data.validation_errors) errors -
- #end if #end if #if $len($data.children)>0
diff --git a/templates/tool_menu.tmpl b/templates/tool_menu.tmpl index 7ecae36433e..0a5246eee0c 100644 --- a/templates/tool_menu.tmpl +++ b/templates/tool_menu.tmpl @@ -73,6 +73,7 @@ div#footer { diff --git a/tool_conf.xml.sample b/tool_conf.xml.sample index 53bf5b93519..5151d04bb2e 100644 --- a/tool_conf.xml.sample +++ b/tool_conf.xml.sample @@ -9,6 +9,7 @@ +
diff --git a/tools/validation/fix_errors.py b/tools/validation/fix_errors.py new file mode 100644 index 00000000000..60e18ef22d5 --- /dev/null +++ b/tools/validation/fix_errors.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +""" +Fix errors in a dataset. +For now, only removing erroneous lines is supported. + +usage: %prog input errorsfile output + -x, --ext: dataset extension (type) + -m, --methods=N: comma separated list of repair methods +""" +from cookbook import doc_optparse +from galaxy import model +from fileinput import FileInput +from galaxy import util + +def main(): + options, args = doc_optparse.parse( __doc__ ) + + try: + if options.methods: methods = options.methods.split(",") + ext = options.ext + except: + doc_optparse.exception() + + in_file = open(args[0], "r") + error_file = open(args[1], "r") + out_file = open(args[2], "w") + + # string_to_object errors + error_list = util.string_to_object(error_file.read()) + # index by error type and then by line number + error_lines = {} + error_types = {} + for error in error_list: + if error.linenum: + if error.linenum in error_lines: + error_lines[error.linenum].append(error) + else: + error_lines[error.linenum] = [error] + error_type = error.__class__.__name__ + if error_type in error_types: + error_types[error_type].append(error) + else: + error_types[error_type] = [error] + + linenum = 0 + for line in in_file: + linenum += 1 + # write unless + if "lines" in methods: + if linenum in error_lines: + line = None + # other processing here? + if line: + out_file.write(line) + +if __name__ == "__main__": + main() diff --git a/tools/validation/fix_errors.xml b/tools/validation/fix_errors.xml new file mode 100644 index 00000000000..50c3095d200 --- /dev/null +++ b/tools/validation/fix_errors.xml @@ -0,0 +1,22 @@ + + \ No newline at end of file diff --git a/tools/validation/fix_errors_code.py b/tools/validation/fix_errors_code.py new file mode 100644 index 00000000000..c1182443ec6 --- /dev/null +++ b/tools/validation/fix_errors_code.py @@ -0,0 +1,49 @@ +# runs after the job (and after the default post-filter) + +import pkg_resources +pkg_resources.require( "bx-python" ) + +from galaxy import datatypes, jobs, util +# needed to reference ParseError types, is this bad? +from bx.tabular.io import * +from bx.intervals.io import * +import sys, tempfile, os + +def validate(incoming): + """Validator""" + #raise Exception, 'not quite right' + pass + +def exec_before_job( trans, inp_data, out_data, param_dict, tool=None): + """Build a temp file with errors in it""" + errors = [] + for name, data in inp_data.items(): + validation_errors = data.validation_errors + for error in validation_errors: + # build dummy class + try: + temp = eval(error.err_type)() + except: + temp = object() + # stuff attributes + temp.__dict__ = util.string_to_object( error.attributes ) + errors.append(temp) + # There *should* only be 1 input, so we assume there is and continue + # base64 pickel + errors_str = util.object_to_string( errors ) + # write + database_tmp = "./database/tmp" # globaly visible path + error_file = tempfile.NamedTemporaryFile(mode="w", dir=database_tmp, suffix=".b64") + error_file_name = error_file.name + error_file.close() + error_file = open(error_file_name, "w") + error_file.write(errors_str) + error_file.close() + param_dict["errorsfile"] = error_file_name + + +def exec_after_process( app, inp_data, out_data, param_dict, tool=None, stdout=None, stderr=None): + # in a perfect world, changes to param_dict would persist + # for now, unlink from tool + # os.unlink(param_dict["errorsfile"]) + pass diff --git a/scripts/validate.py b/tools/validation/validate.py similarity index 100% rename from scripts/validate.py rename to tools/validation/validate.py