From 3db7688a5628e459f2c02c73338f39e457495580 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 10 Dec 2013 12:59:29 -0500 Subject: [PATCH] Add Functional Testing support for locally and ToolShed installed Data Managers. --- lib/galaxy/tools/__init__.py | 4 + lib/galaxy/tools/data_manager/manager.py | 11 +- run_functional_tests.sh | 9 ++ scripts/functional_tests.py | 31 +++- test/functional/test_data_managers.py | 149 ++++++++++++++++++ test/functional/test_toolbox.py | 6 + .../data_manager_files/test_data_manager.tar | Bin 13312 -> 1932 bytes 7 files changed, 202 insertions(+), 8 deletions(-) create mode 100644 test/functional/test_data_managers.py diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 4a4604e7173..02e11bd18a2 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -394,6 +394,10 @@ class ToolBox( object, Dictifiable ): return tool #No tool matches by version, simply return the first available tool found return rval[0] + #We now likely have a Toolshed guid passed in, but no supporting database entries + #If the tool exists by exact id and is loaded then provide exact match within a list + if tool_id in self.tools_by_id: + return[ self.tools_by_id[ tool_id ] ] return None def get_loaded_tools_by_lineage( self, tool_id ): diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 2d3b500e8a1..decee9e1e2b 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -133,7 +133,16 @@ class DataManager( object ): self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision ) #get tool_shed repo id tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.data_managers.app, tool_shed, repository_name, repository_owner, installed_changeset_revision ) - tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) + if tool_shed_repository is None: + log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' ) + #we'll set tool_path manually here from shed_conf_file + tool_shed_repository_id = None + try: + tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path ) + except Exception, e: + log.error( 'Error determining tool_path for Data Manager during testing: %s', e ) + else: + tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) #use shed_conf_file to determine tool_path shed_conf_file = elem.get( "shed_conf_file", None ) if shed_conf_file: diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 33fd02048df..ac445323aa4 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -16,6 +16,7 @@ elif [ $1 = 'help' ]; then echo "'run_functional_tests.sh -workflow test.xml' for running a workflow test case as defined by supplied workflow xml test file" echo "'run_functional_tests.sh -framework' for running through example tool tests testing framework features in test/functional/tools" echo "'run_functional_tests.sh -framework -id toolid' for testing one framework tool (in test/functional/tools/) with id 'toolid'" + echo "'run_functional_tests.sh -data_managers -id data_manager_id' for testing one Data Manager with id 'data_manager_id'" elif [ $1 = '-id' ]; then python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$2 --with-nosehtml --html-report-file run_functional_tests.html elif [ $1 = '-sid' ]; then @@ -51,6 +52,14 @@ elif [ $1 = '-toolshed' ]; then fi elif [ $1 = '-workflow' ]; then python ./scripts/functional_tests.py -v functional.test_workflow:WorkflowTestCase --with-nosehtml --html-report-file ./test/tool_shed/run_functional_tests.html -workflow $2 +elif [ $1 = '-data_managers' ]; then + if [ ! $2 ]; then + python ./scripts/functional_tests.py -v functional.test_data_managers --with-nosehtml --html-report-file run_functional_tests.html -data_managers + elif [ $2 = '-id' ]; then + python ./scripts/functional_tests.py -v functional.test_data_managers:TestForDataManagerTool_$3 --with-nosehtml --html-report-file run_functional_tests.html -data_managers + else + python ./scripts/functional_tests.py -v functional.test_data_managers --with-nosehtml --html-report-file run_functional_tests.html -data_managers + fi elif [ $1 = '-framework' ]; then if [ ! $2 ]; then python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -framework diff --git a/scripts/functional_tests.py b/scripts/functional_tests.py index 5744f7cf10f..33dacd812db 100644 --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -335,6 +335,11 @@ def main(): except OSError: pass + #Data Manager testing temp path + #For storing Data Manager outputs and .loc files so that real ones don't get clobbered + data_manager_test_tmp_path = tempfile.mkdtemp( prefix='data_manager_test_tmp', dir=galaxy_test_tmp_dir ) + galaxy_data_manager_data_path = tempfile.mkdtemp( prefix='data_manager_tool-data', dir=data_manager_test_tmp_path ) + # ---- Build Application -------------------------------------------------- master_api_key = os.environ.get( "GALAXY_TEST_MASTER_API_KEY", default_galaxy_master_key ) app = None @@ -360,6 +365,7 @@ def main(): tool_config_file=tool_config_file, tool_data_table_config_path=tool_data_table_config_path, tool_path=tool_path, + galaxy_data_manager_data_path=galaxy_data_manager_data_path, tool_parse_help=False, update_integrated_tool_panel=False, use_heartbeat=False, @@ -469,13 +475,24 @@ def main(): functional.test_workflow.WorkflowTestCase.workflow_test_file = workflow_test functional.test_workflow.WorkflowTestCase.master_api_key = master_api_key functional.test_workflow.WorkflowTestCase.user_api_key = os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ) - import functional.test_toolbox - functional.test_toolbox.toolbox = app.toolbox - functional.test_toolbox.build_tests( - testing_shed_tools=testing_shed_tools, - master_api_key=master_api_key, - user_api_key=os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ), - ) + data_manager_test = __check_arg( '-data_managers', param=False ) + if data_manager_test: + import functional.test_data_managers + functional.test_data_managers.data_managers = app.data_managers #seems like a hack... + functional.test_data_managers.build_tests( + tmp_dir=data_manager_test_tmp_path, + testing_shed_tools=testing_shed_tools, + master_api_key=master_api_key, + user_api_key=os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ), + ) + else: #when testing data managers, do not test toolbox + import functional.test_toolbox + functional.test_toolbox.toolbox = app.toolbox + functional.test_toolbox.build_tests( + testing_shed_tools=testing_shed_tools, + master_api_key=master_api_key, + user_api_key=os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ), + ) test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) result = run_tests( test_config ) diff --git a/test/functional/test_data_managers.py b/test/functional/test_data_managers.py new file mode 100644 index 00000000000..931aefd7f4f --- /dev/null +++ b/test/functional/test_data_managers.py @@ -0,0 +1,149 @@ +import new +import sys +import tempfile +import os.path +import shutil +from test_toolbox import ToolTestCase +from base.interactor import build_interactor, stage_data_in_history +import logging +log = logging.getLogger( __name__ ) + +data_managers = None + +class DataManagerToolTestCase( ToolTestCase ): + """Test case that runs Data Manager tests based on a `galaxy.tools.test.ToolTest`""" + + def do_it( self, testdef ): + """ + Run through a tool test case. + """ + shed_tool_id = self.shed_tool_id + + self.__handle_test_def_errors( testdef ) + + galaxy_interactor = self.__galaxy_interactor( testdef ) + + test_history = galaxy_interactor.new_history() #history where inputs will be put, if any + + stage_data_in_history( galaxy_interactor, testdef.test_data(), test_history, shed_tool_id ) + + data_list = galaxy_interactor.run_tool( testdef, test_history ) #test_history will have inputs only, outputs are placed in the specialized data manager history + + #FIXME: Move history determination and switching into the interactor + data_manager_history = None + for assoc in reversed( test_history.user.data_manager_histories ): + if not assoc.history.deleted: + data_manager_history = assoc.history + break + self.switch_history( id=self.security.encode_id( data_manager_history.id ) ) + data_list = self.get_history_as_data_list() + #end + + self.assertTrue( data_list ) + + self.__verify_outputs( testdef, data_manager_history, shed_tool_id, data_list, galaxy_interactor ) + + self.switch_history( id=self.security.encode_id( test_history.id ) ) + + galaxy_interactor.delete_history( test_history ) + + + #Methods below are from test_toolbox.py:ToolTestCase, but are not inherited properly + + def __galaxy_interactor( self, testdef ): + return build_interactor( self, testdef.interactor ) + + def __handle_test_def_errors(self, testdef): + # If the test generation had an error, raise + if testdef.error: + if testdef.exception: + raise testdef.exception + else: + raise Exception( "Test parse failure" ) + + def __verify_outputs( self, testdef, history, shed_tool_id, data_list, galaxy_interactor ): + maxseconds = testdef.maxseconds + + for output_index, output_tuple in enumerate(testdef.outputs): + # Get the correct hid + name, outfile, attributes = output_tuple + try: + output_data = data_list[ name ] + except (TypeError, KeyError): + # Legacy - fall back on ordered data list access if data_list is + # just a list (case with twill variant or if output changes its + # name). + if hasattr(data_list, "values"): + output_data = data_list.values()[ output_index ] + else: + output_data = data_list[ len(data_list) - len(testdef.outputs) + output_index ] + self.assertTrue( output_data is not None ) + try: + galaxy_interactor.verify_output( history, output_data, outfile, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) + except Exception: + for stream in ['stdout', 'stderr']: + stream_output = galaxy_interactor.get_job_stream( history, output_data, stream=stream ) + print >>sys.stderr, self._format_stream( stream_output, stream=stream, format=True ) + raise + +def build_tests( tmp_dir=None, testing_shed_tools=False, master_api_key=None, user_api_key=None ): + """ + If the module level variable `data_managers` is set, generate `DataManagerToolTestCase` + classes for all of its tests and put them into this modules globals() so + they can be discovered by nose. + """ + + if data_managers is None: + log.warning( 'data_managers was not set for Data Manager functional testing. Will not test.' ) + return + + # Push all the data_managers tests to module level + G = globals() + + # Eliminate all previous tests from G. + for key, val in G.items(): + if key.startswith( 'TestForDataManagerTool_' ): + del G[ key ] + + #first we will loop through data table loc files and copy them to temporary location, then swap out filenames: + for data_table_name, data_table in data_managers.app.tool_data_tables.get_tables().iteritems(): + for filename, value in list( data_table.filenames.items() ): + new_filename = tempfile.NamedTemporaryFile( prefix=os.path.basename( filename ), dir=tmp_dir ).name + try: + shutil.copy( filename, new_filename ) + except IOError, e: + log.warning( "Failed to copy '%s' to '%s', will create empty file at '%s': %s", filename, new_filename, new_filename, e ) + open( new_filename, 'wb' ).close() + if 'filename' in value: + value[ 'filename' ] = new_filename + del data_table.filenames[ filename ] #remove filename:value pair + data_table.filenames[ new_filename ] = value #add new value by + + for i, ( data_manager_id, data_manager ) in enumerate( data_managers.data_managers.iteritems() ): + tool = data_manager.tool + if not tool: + log.warning( "No Tool has been specified for Data Manager: %s", data_manager_id ) + tool_data_tables = data_manager.data_managers.app.tool_data_tables + if tool.tests: + ##fixme data_manager.tool_shed_repository_info_dict should be filled when is toolshed based + shed_tool_id = None if not testing_shed_tools else tool.id + # Create a new subclass of ToolTestCase, dynamically adding methods + # named test_tool_XXX that run each test defined in the tool config. + name = "TestForDataManagerTool_" + data_manager_id.replace( ' ', '_' ) + baseclasses = ( DataManagerToolTestCase, ) + namespace = dict() + for j, testdef in enumerate( tool.tests ): + def make_test_method( td ): + def test_tool( self ): + self.do_it( td ) + return test_tool + test_method = make_test_method( testdef ) + test_method.__doc__ = "%s ( %s ) > %s" % ( tool.name, tool.id, testdef.name ) + namespace[ 'test_tool_%06d' % j ] = test_method + namespace[ 'shed_tool_id' ] = shed_tool_id + namespace[ 'master_api_key' ] = master_api_key + namespace[ 'user_api_key' ] = user_api_key + # The new.classobj function returns a new class object, with name name, derived + # from baseclasses (which should be a tuple of classes) and with namespace dict. + new_class_obj = new.classobj( name, baseclasses, namespace ) + G[ name ] = new_class_obj diff --git a/test/functional/test_toolbox.py b/test/functional/test_toolbox.py index 6739807ff1f..0e54d485b82 100644 --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -2,11 +2,14 @@ import new import sys from base.twilltestcase import TwillTestCase from base.interactor import build_interactor, stage_data_in_history +from galaxy.tools import DataManagerTool import logging log = logging.getLogger( __name__ ) toolbox = None +#Do not test Data Managers as part of the standard Tool Test Framework. +TOOL_TYPES_NO_TEST = ( DataManagerTool, ) class ToolTestCase( TwillTestCase ): """Abstract test case that runs tests based on a `galaxy.tools.test.ToolTest`""" @@ -88,6 +91,9 @@ def build_tests( testing_shed_tools=False, master_api_key=None, user_api_key=Non for i, tool_id in enumerate( toolbox.tools_by_id ): tool = toolbox.get_tool( tool_id ) + if isinstance( tool, TOOL_TYPES_NO_TEST ): + #We do not test certain types of tools (e.g. Data Manager tools) as part of ToolTestCase + continue if tool.tests: shed_tool_id = None if not testing_shed_tools else tool.id # Create a new subclass of ToolTestCase, dynamically adding methods diff --git a/test/tool_shed/test_data/1460_files/data_manager_files/test_data_manager.tar b/test/tool_shed/test_data/1460_files/data_manager_files/test_data_manager.tar index c36282ccb299929215c779101a41005295ce009a..b4b09a8d7e4b1372bd80e111d43ecc6d6597ae07 100644 GIT binary patch literal 1932 zcmV;72XpuziwFRIKBrOu1MOOSZ`(K$&)$F4r=w8y zdVxS&RIH6&D^h6nnvaO?GEr`>Kg-44`T*KM>8p!NK$9UK&llU%UkljOmAqFN9as3~G?f8Tf zFD41Oq51O|!&m6PRYUzZT3Y}6!K(hfQJMtympFRTK7>4kW8bTPw=IlA=wImGZM8d6 z|E+qva{#p`<>DUvpX1z&xM1*0@7;-sW_N1}q7;OwgB4{zz@5%uK% zBdTSDk4;c4dN~dHrnpA|ny}dx+sEl01wP3{Ypl1u8dha#k&o#4>#6K!)^p0|`x^-B zzM&k3gbC`er#}wf4gU4fyBK^Jc<%>SgC8${RS&-mKKx^53c@Ik*;e&Bucs)~nX~9$ zk-ukLjvUxMWk|^5IZ0SVc$g;VyO?vnmbc9=9?8mF7ez9XkLEcUGD-7MBzulBhc%mZ z){VC3EDgN$yOmgcZmd2F=X?hLx1C=v2N&-zpMmrOJL3OJ{@3WVMEtLd7CQd#12#I- znKZO#`x1C${)h8_r{l`}Pt?@;-##GpFXs3E3jJgL->BD6|87I;e;-i#ujE>bPXL>d zIE&~L^sy`cHyR85cRF?H|DAeE$N#;chf1F({*O+`VU)%(O}IB2le}*pR#SVG{;+e* zl}-V3wo@c}3Mz}5*I2fZX)hLvFNZ}udWZu%N=J)iT&AIzMU^+tlPdisQH}q*yXI>D-v@1-O-gER<$#u=X&hv>quLjRTY-dEB;Uz%Ty zr$sgY&+;%P`2^Q0?B7}XutPi^yZpc2UgZDHrYrp4Z8usi?f?70)Lh1RW_Drv@VwuJ zPgAwR)diEKF@<3fM*J*HY+SiIg>pGf6bIy%UcC9~@}qpK7URlBUXg@<6x0iSS%m9q zqGSMm$g+q`Fjs&e&4IuVf(Tr^fu97;+4XGp((?MNq3Zve`h1!`cCY_57XI%xI_Uqk zhW7ux;Jd%QGzS_lh_(ISt(!X8&YkivKg&pUhpvlP?^pZ&h9p+)C>Vklm>`_ruwx;| zl3~1f@?|N*`DjSYM6Go-9|`%>ME)Z=doBhzwsT7SqvNtawe{%vVNOS41QXSvp!o@` za{Ew{d?qYD_4a&N^ z#?x%s!~9c23PyEN`63Ywrwc>aW_;pIP!-PVfpeXx7=Yg`Xo>Q%1l=}*5z77*k^DL+0zi+C9#XP?!afRii4Cd?U{3OoI zgzZx%?AZ2@dl$$>$y1n!X{Nf7fkA%tV38667xRw^5gF2`Z{o6uj75a8@FtOWD4pu* zqLNg#b!B5l9Y(?^+&33=ujCNY02uwFpvj1S1;NnYB=`0LS7A1&^_=Rs%i~yV?Q~a3 ze0lgukctUkGc*}K%>~&NNS3hsQbClWS(|zoSghqX6hjaKc(#C#4z53I#u>se4BLiLBuus2#Yt4Kb!&(&u2LS3X!qmU+ z{f0xykspcN#3}Y+1BoqhI2``S84hR0uD^KpkK;Jqb{k%(?~eX71T6y5KY#jaK^ww4Yrv93T^tr6}5%MGp>#WRuAQ6!|>VB=G|eQ2wEs?E08V=7lU` z*F1fm&474pyZ&v5*1y%>um88%Cd=#ZjpArxe+h$~4vy=OihZyCH#%q>Q2%l58|_Y4 z>3_S~>AnDGr&8R*>~F6BZdv`snF07&dIvI&18?F7yl<9i>yiuUN7r8YrP?|s_~X90 zsnyi%?}m_U7;uPKxWe00y)|PZDRrdIaVC>Y-f>ee)yEyhK^8_i$4@NCxC!+ty7BOa z&$ggba&AGEFgZ0ri|D0%(l_xp0cgUyC?s|mf8ro#3BsU&UYRaUE&u0=I7>(T?A5Ir zX71MWXIi&1Ve#9=)9so|7&3v}U)}yVcsuyd2k(6Fe&D?uTn>J`_(ebbJb3@FxrvMk zj`_yrt-rb@Q0Mp~A78V-XI^zRv3uHxktb6YiGWEzj&SIQg~LI=PPX+SHp1kt(G_cv zMI$AddB%o9&DX83k*?3qt?OO5+~U?(H?6BrUfSTAHh+_!#b?IyGb-mjChRG%?fNel zgY$P6PvKG3x5EBov$OF3THQ9<|0bRod;4F7>n_@DQuGsL#Gl9cLm_c}xAFhz{O@)f zxc)8dvG@OXvA$J2aQ*Ro|3j?5y8pFXO^4RM(b}(n)!OZ%U9W%1*IHZwY{9}L;5!(k z%Ko=n3;W;gHkJN&o9(^*-vRQ~EYLqhAG~nPP^JG)XQBTX_vwJsY<8T+-v9d++v(u_)EA)jUPVHsfOs+i9wu^T9FPjX z^v5sN-S?$Z-ftn(evEI2K-vFKQa@zrjA9k`?;?IUAilkfRr-(GUc~<}{*U_K=(O4x z|KG>|w#WFfl(#ylr@a8P2l;azgIGOvk>KO4sq6s$GDRG67-oJTPyEQH$W;wY9ZtjM z3A^FvuYbDuVB%Y{Bt<7Li~Rp(-1En(iQ;NVKY+0>l7P*~SAa>J0)ror37o%%pD;u> zznaeEZt%d$KS}OkEX*qcR+w1?=BK-*Z@BZwpiW@pk5$nAu@gKG4YALJ;ytsilEPtv8x zrjXSBccSt?M4lcVm56LM9(zcJt%qQpOspFIKT$4_@@DX9j2X^-ihy_<4ovV))|ANUzy4wg(d9c7w;SJ8^5HI5@V_TY+Uj5I$oEFFk#8gi0-(%d=YxFf( zR*NIto1cdk903zBnQ9$SQMbv|{+#-f;|~5W+GzZ1;9nCdH8M&vuNM-Xd|khNI_WlR%DvrjZj=Q6y%654so}Qg@reg#YWh)`S3ylcl4eX z^a@}OdI%clKrl?O`_Uvu3j)fbfYMBAE5l0qEEXHC8nKW*&ef7Oe(0}59u z6;ASMy#M{wy8(CVb+lz|*jAI<{jsOkFmlYg#l@$}#160vC@*awxFJ zSw0pi51!Lhh=`>RB??kh%q!^g+K(hpla%BCzNsA+vtY7Zjg-vb?obA-ekm$(<|a05 zGA9_mP*$%DLn$|Af-3GHDe{U9Kf*$!aw3C{j~pl+3CW8|B%VJe1Z>EIzDZ#bWs8^) z{&l2&sc{zx4a$jVRvJRxTUpe+3QPS&&{c_3-(2v$RL5jI0l_i65{;J2mD~pzVk~Bj z-J5fY!mM#~>*aa7`mimdpeB=pP(W@fRw0wiRb=GOnR$s;Q6rOca@EEtb5+kPRW%#n z&M~=*KCKjh@Mzmi~zMj*sC(oPn6eSul<)2^qT9QY95AfEVF~0zF)?&~*ze i8w{<>buB37>(Kl>OJUOvY~!BA9)Udqdj$T<2>c(uDJ-x6