From 9a9a5058d22d3c0da651a3a89e155c20433d60a1 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 16 Sep 2014 14:36:37 -0400 Subject: [PATCH 01/11] Fix config moves for the Tool Shed. --- lib/galaxy/webapps/tool_shed/config.py | 48 ++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/config.py b/lib/galaxy/webapps/tool_shed/config.py index 6af36108933..05fc22ebd7d 100644 --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -27,6 +27,10 @@ class Configuration( object ): def __init__( self, **kwargs ): self.config_dict = kwargs self.root = kwargs.get( 'root_dir', '.' ) + + # Resolve paths of other config files + self.__parse_config_file_options( kwargs ) + # Collect the umask and primary gid from the environment self.umask = os.umask( 077 ) # get the current umask os.umask( self.umask ) # can't get w/o set, so set it back @@ -41,7 +45,6 @@ class Configuration( object ): self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root ) self.cookie_path = kwargs.get( "cookie_path", "/" ) self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) ) - self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) # Tool stuff @@ -51,8 +54,6 @@ class Configuration( object ): self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) self.tool_secret = kwargs.get( "tool_secret", "" ) self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "shed-tool-data" ), os.getcwd() ) - self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root ) - self.shed_tool_data_table_config = resolve_path( kwargs.get( 'shed_tool_data_table_config', 'shed_tool_data_table_conf.xml' ), self.root ) self.integrated_tool_panel_config = resolve_path( kwargs.get( 'integrated_tool_panel_config', 'integrated_tool_panel.xml' ), self.root ) self.builds_file_path = resolve_path( kwargs.get( "builds_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'builds.txt') ), self.root ) self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared','ucsc','chrom') ), self.root ) @@ -136,6 +137,47 @@ class Configuration( object ): self.citation_cache_data_dir = resolve_path( kwargs.get( "citation_cache_data_dir", "database/tool_shed_citations/data" ), self.root ) self.citation_cache_lock_dir = resolve_path( kwargs.get( "citation_cache_lock_dir", "database/tool_shed_citations/locks" ), self.root ) + def __parse_config_file_options( self, kwargs ): + defaults = dict( + datatypes_config_file = [ 'datatypes_conf.xml', 'config/datatypes_conf.xml', 'config/datatypes_conf.xml.sample' ], + shed_tool_data_table_config = [ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ], + ) + + listify_defaults = dict( + tool_data_table_config_path = [ 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample' ], + ) + + for var, defaults in defaults.items(): + if kwargs.get( var, None ) is not None: + path = kwargs.get( var ) + else: + for default in defaults: + if os.path.exists( resolve_path( default, self.root ) ): + path = default + break + else: + path = defaults[-1] + setattr( self, var, resolve_path( path, self.root ) ) + + for var, defaults in listify_defaults.items(): + paths = [] + if kwargs.get( var, None ) is not None: + paths = listify( kwargs.get( var ) ) + else: + for default in defaults: + for path in listify( default ): + if not os.path.exists( resolve_path( path, self.root ) ): + break + else: + paths = listify( default ) + break + else: + paths = listify( defaults[-1] ) + setattr( self, var, [ resolve_path( x, self.root ) for x in paths ] ) + + # Backwards compatibility for names used in too many places to fix + self.datatypes_config = self.datatypes_config_file + def get( self, key, default ): return self.config_dict.get( key, default ) From 43cf0421628bd94d04cd83cb258c06df8b726fea Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 16 Sep 2014 14:48:39 -0400 Subject: [PATCH 02/11] Remove newly redundant (and broken) test case. That case was testing rename post job actions without replacements - last week I added a more expansive one with replacements (test_run_with_pja). --- test/api/test_workflows.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 75a7594586c..de32f74c434 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -558,21 +558,6 @@ class WorkflowsApiTestCase( api.ApiTestCase ): for step in usage_details[ "steps" ]: self._assert_has_keys( step, "workflow_step_id", "order_index", "id" ) - @skip_without_tool( "cat1" ) - def test_post_job_action( self ): - """ Tests both import and execution of post job actions. - """ - workflow = self.workflow_populator.load_workflow( name="test_for_pja_run", add_pja=True ) - workflow_request, history_id = self._setup_workflow_run( workflow ) - run_workflow_response = self._post( "workflows", data=workflow_request ) - self._assert_status_code_is( run_workflow_response, 200 ) - self.dataset_populator.wait_for_history( history_id, assert_ok=True ) - time.sleep(.1) # Give another little bit of time for rename (needed?) - contents = self._get( "histories/%s/contents" % history_id ).json() - # loading workflow with add_pja=True causes workflow output to be - # renamed to 'the_new_name'. - assert "the_new_name" in map( lambda hda: hda[ "name" ], contents ) - def _invocation_details( self, workflow_id, invocation_id ): invocation_details_response = self._get( "workflows/%s/usage/%s" % ( workflow_id, invocation_id ) ) self._assert_status_code_is( invocation_details_response, 200 ) From 55c1ac6d5398a72ccd297b4fc8493eea17df5427 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 16 Sep 2014 14:48:39 -0400 Subject: [PATCH 03/11] Update workflow API tests for inability to decode ids being a 400 instead of 404. Sort of torn on what this should actually be I guess. --- test/api/test_workflows.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index de32f74c434..95f616c6afc 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -26,9 +26,9 @@ class WorkflowsApiTestCase( api.ApiTestCase ): self.dataset_populator = DatasetPopulator( self.galaxy_interactor ) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor ) - def test_show_invalid_is_404( self ): - show_response = self._get( "workflow/%s" % self._random_key() ) - self._assert_status_code_is( show_response, 404 ) + def test_show_invalid_key_is_400( self ): + show_response = self._get( "workflows/%s" % self._random_key() ) + self._assert_status_code_is( show_response, 400 ) def test_cannot_show_private_workflow( self ): workflow_id = self.workflow_populator.simple_workflow( "test_not_importportable" ) @@ -134,12 +134,12 @@ class WorkflowsApiTestCase( api.ApiTestCase ): run_workflow_response = self._post( "workflows", data=workflow_request ) self._assert_status_code_is( run_workflow_response, 403 ) - def test_404_on_invalid_workflow( self ): + def test_400_on_invalid_workflow_id( self ): workflow = self.workflow_populator.load_workflow( name="test_for_run_does_not_exist" ) workflow_request, history_id = self._setup_workflow_run( workflow ) workflow_request[ "workflow_id" ] = self._random_key() run_workflow_response = self._post( "workflows", data=workflow_request ) - self._assert_status_code_is( run_workflow_response, 404 ) + self._assert_status_code_is( run_workflow_response, 400 ) def test_cannot_run_against_other_users_history( self ): workflow = self.workflow_populator.load_workflow( name="test_for_run_does_not_exist" ) From 4132fc85988e7ca071fc2b09fa7d651bb8a3aa24 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 16 Sep 2014 14:48:39 -0400 Subject: [PATCH 04/11] Fix library API test for format field redefinition in 9d152ed. --- test/api/test_libraries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index 6060e35d890..1a3bee28dfa 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -43,7 +43,7 @@ class LibrariesApiTestCase( api.ApiTestCase, TestsDatasets ): library_dataset = show().json() self._assert_has_keys( library_dataset, "peek", "data_type" ) assert library_dataset[ "peek" ].find("create_test") >= 0 - assert library_dataset[ "data_type" ] == "txt" + assert library_dataset[ "file_ext" ] == "txt", library_dataset[ "file_ext" ] def _create_folder( self, library ): create_data = dict( From c7acc3723f90ed501e557ce3ea440ebb00f83a05 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 16 Sep 2014 14:55:38 -0400 Subject: [PATCH 05/11] Rename tool_shed_wsgi.ini.sample to tool_shed_wsgi.ini, fix static and other errors I created when moving it to config/ in the first place. --- ...d_wsgi.ini.sample => tool_shed.ini.sample} | 14 ++++++------- lib/galaxy/model/orm/scripts.py | 4 ++-- lib/galaxy/webapps/tool_shed/app.py | 2 +- lib/galaxy/webapps/tool_shed/buildapp.py | 11 +++++----- .../bootstrap_tool_shed.sh | 20 ++++++++++--------- .../bootstrap_tool_shed/bootstrap_util.py | 4 ++-- run_tool_shed.sh | 13 +++++++++++- 7 files changed, 41 insertions(+), 27 deletions(-) rename config/{tool_shed_wsgi.ini.sample => tool_shed.ini.sample} (93%) diff --git a/config/tool_shed_wsgi.ini.sample b/config/tool_shed.ini.sample similarity index 93% rename from config/tool_shed_wsgi.ini.sample rename to config/tool_shed.ini.sample index f83eed3b8cb..029f437e82a 100644 --- a/config/tool_shed_wsgi.ini.sample +++ b/config/tool_shed.ini.sample @@ -105,10 +105,10 @@ use_new_layout = true #message_box_class = info # Serving static files (needed if running standalone) -static_enabled = True -static_cache_time = 360 -static_dir = %(here)s/static/ -static_images_dir = %(here)s/static/images -static_favicon_dir = %(here)s/static/favicon.ico -static_scripts_dir = %(here)s/static/scripts/ -static_style_dir = %(here)s/static/june_2007_style/blue +#static_enabled = True +#static_cache_time = 360 +#static_dir = static/ +#static_images_dir = static/images +#static_favicon_dir = static/favicon.ico +#static_scripts_dir = static/scripts/ +#static_style_dir = static/style/blue diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index 1161fb5230a..59f31dcd4c4 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -31,7 +31,7 @@ DATABASE = { "tool_shed": { 'repo': 'lib/galaxy/webapps/tool_shed/model/migrate', - 'config_file': 'tool_shed_wsgi.ini', + 'config_file': 'config/tool_shed.ini', 'default_sqlite_file': './database/community.sqlite', }, "install": @@ -79,7 +79,7 @@ def get_config( argv, cwd=None ): ... p.add_section('app:main') ... p.set('app:main', property, value) ... with open(os.path.join(config_dir, path), 'w') as f: p.write(f) - >>> write_ini('tool_shed_wsgi.ini', 'database_connection', 'sqlite:///pg/testdb1') + >>> write_ini('tool_shed.ini', 'database_connection', 'sqlite:///pg/testdb1') >>> config = get_config(['manage_db.py', 'tool_shed'], cwd=config_dir) >>> config['repo'] 'lib/galaxy/webapps/tool_shed/model/migrate' diff --git a/lib/galaxy/webapps/tool_shed/app.py b/lib/galaxy/webapps/tool_shed/app.py index 4cf8636db2e..5b411312db7 100644 --- a/lib/galaxy/webapps/tool_shed/app.py +++ b/lib/galaxy/webapps/tool_shed/app.py @@ -20,7 +20,7 @@ class UniverseApplication( object ): def __init__( self, **kwd ): print >> sys.stderr, "python path is: " + ", ".join( sys.path ) self.name = "tool_shed" - # Read the tool_shed_wsgi.ini configuration file and check for errors. + # Read the tool_shed.ini configuration file and check for errors. self.config = config.Configuration( **kwd ) self.config.check() config.configure_logging( self.config ) diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index c2ce0c39b4b..14ab0b0337f 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -220,11 +220,12 @@ def wrap_in_static( app, global_conf, **local_conf ): # Send to dynamic app by default urlmap["/"] = app # Define static mappings from config - urlmap["/static"] = Static( conf.get( "static_dir" ), cache_time ) - urlmap["/images"] = Static( conf.get( "static_images_dir" ), cache_time ) - urlmap["/static/scripts"] = Static( conf.get( "static_scripts_dir" ), cache_time ) - urlmap["/static/style"] = Static( conf.get( "static_style_dir" ), cache_time ) - urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir" ), cache_time ) + urlmap["/static"] = Static( conf.get( "static_dir", "./static/" ), cache_time ) + urlmap["/images"] = Static( conf.get( "static_images_dir", "./static/images" ), cache_time ) + urlmap["/static/scripts"] = Static( conf.get( "static_scripts_dir", "./static/scripts/" ), cache_time ) + urlmap["/static/style"] = Static( conf.get( "static_style_dir", "./static/style/blue" ), cache_time ) + urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir", "./static/favicon.ico" ), cache_time ) + urlmap["/robots.txt"] = Static( conf.get( "static_robots_txt", "./static/robots.txt" ), cache_time ) # URL mapper becomes the root webapp return urlmap diff --git a/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_tool_shed.sh b/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_tool_shed.sh index bdb85bb3a83..279813c7cf6 100755 --- a/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_tool_shed.sh +++ b/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_tool_shed.sh @@ -1,8 +1,10 @@ #!/bin/bash +: ${TOOL_SHED_CONFIG_FILE:=config/tool_shed.ini.sample} + stop_err() { echo $1 - python ./scripts/paster.py serve tool_shed_wsgi.ini --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --stop-daemon + python ./scripts/paster.py serve ${TOOL_SHED_CONFIG_FILE} --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --stop-daemon exit 1 } @@ -14,7 +16,7 @@ fi log_file="lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap.log" -database_result=`python ./lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py --execute check_db --config_file tool_shed_wsgi.ini` +database_result=`python ./lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py --execute check_db --config_file ${TOOL_SHED_CONFIG_FILE}` if [ $? -ne 0 ] ; then stop_err "Unable to bootstrap tool shed. $database_result" @@ -31,8 +33,8 @@ else fi if [ $? -eq 0 ] ; then - user_auth=`python ./lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py --execute admin_user_info --config_file tool_shed_wsgi.ini` - local_shed_url=`python ./lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py --execute get_url --config_file tool_shed_wsgi.ini` + user_auth=`python ./lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py --execute admin_user_info --config_file ${TOOL_SHED_CONFIG_FILE}` + local_shed_url=`python ./lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py --execute get_url --config_file ${TOOL_SHED_CONFIG_FILE}` fi admin_user_name=`echo $user_auth | awk 'BEGIN { FS="__SEP__" } ; { print \$1 }'` @@ -41,18 +43,18 @@ admin_user_password=`echo $user_auth | awk 'BEGIN { FS="__SEP__" } ; { print \$3 echo -n "Creating user '$admin_user_name' with email address '$admin_user_email'..." -python lib/tool_shed/scripts/bootstrap_tool_shed/create_user_with_api_key.py tool_shed_wsgi.ini >> $log_file +python lib/tool_shed/scripts/bootstrap_tool_shed/create_user_with_api_key.py ${TOOL_SHED_CONFIG_FILE} >> $log_file echo " done." -sed -i "s/#admin_users = user1@example.org,user2@example.org/admin_users = $admin_user_email/" tool_shed_wsgi.ini +sed -i "s/#admin_users = user1@example.org,user2@example.org/admin_users = $admin_user_email/" ${TOOL_SHED_CONFIG_FILE} echo -n "Starting tool shed in order to populate users and categories... " if [ -f tool_shed_bootstrap.pid ] ; then stop_err "A bootstrap process is already running." fi -python ./scripts/paster.py serve tool_shed_wsgi.ini --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --daemon > /dev/null +python ./scripts/paster.py serve ${TOOL_SHED_CONFIG_FILE} --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --daemon > /dev/null shed_pid=`cat tool_shed_bootstrap.pid` @@ -94,6 +96,6 @@ else fi echo "Bootstrap complete, shutting down temporary tool shed process. A log has been saved to tool_shed_bootstrap.log" -python ./scripts/paster.py serve tool_shed_wsgi.ini --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --stop-daemon +python ./scripts/paster.py serve ${TOOL_SHED_CONFIG_FILE} --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --stop-daemon -exit 0 \ No newline at end of file +exit 0 diff --git a/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py b/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py index 4ca3f4387f6..d4f58307290 100755 --- a/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py +++ b/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py @@ -25,7 +25,7 @@ def check_db( config_parser ): db_file = config_parser.get( 'app:main', 'database_file' ) dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file else: - print 'The database configuration setting is missing from the tool_shed_wsgi.ini file. Add this setting before attempting to bootstrap.' + print 'The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap.' exit(1) sa_session = None @@ -122,7 +122,7 @@ def main( args ): return 1 parser = argparse.ArgumentParser() -parser.add_argument( '-c', '--config_file', dest='config', action='store', default='tool_shed_wsgi.ini' ) +parser.add_argument( '-c', '--config_file', dest='config', action='store', default='config/tool_shed.ini.sample' ) parser.add_argument( '-e', '--execute', dest='method', action='store', default='check_db' ) args = parser.parse_args() diff --git a/run_tool_shed.sh b/run_tool_shed.sh index 1d09ad98449..9f6a0b3eaed 100755 --- a/run_tool_shed.sh +++ b/run_tool_shed.sh @@ -12,4 +12,15 @@ if [ $? -eq 0 ] ; then args=`echo $@ | sed "s#-\?-bootstrap_from_tool_shed $tool_shed##"` fi -python ./scripts/paster.py serve tool_shed_wsgi.ini --pid-file=tool_shed_webapp.pid --log-file=tool_shed_webapp.log $args +if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then + if [ -f tool_shed_wsgi.ini ]; then + TOOL_SHED_CONFIG_FILE=tool_shed_wsgi.ini + elif [ -f config/tool_shed.ini ]; then + TOOL_SHED_CONFIG_FILE=config/tool_shed.ini + else + TOOL_SHED_CONFIG_FILE=config/tool_shed.ini.sample + fi + export TOOL_SHED_CONFIG_FILE +fi + +python ./scripts/paster.py serve $TOOL_SHED_CONFIG_FILE --pid-file=tool_shed_webapp.pid --log-file=tool_shed_webapp.log $args From 9857726da5938813211a57225d1ec190a17ce171 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 16 Sep 2014 16:29:31 -0400 Subject: [PATCH 06/11] Update to latest Pulsar client. Updates to Pulsar changeset f8d832fe4896f3fa9e27cc66a2aad7e446b99c2a - see Pulsar (https://github.com/galaxyproject/pulsar) for individual changes. --- lib/pulsar/client/__init__.py | 2 +- lib/pulsar/client/amqp_exchange.py | 18 ++++++-- lib/pulsar/client/transport/__init__.py | 15 +++++-- lib/pulsar/client/transport/curl.py | 3 +- lib/pulsar/client/transport/poster.py | 56 +++++++++++++++++++++++++ lib/pulsar/client/transport/requests.py | 50 ++++++++++++++++++++++ 6 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 lib/pulsar/client/transport/poster.py create mode 100644 lib/pulsar/client/transport/requests.py diff --git a/lib/pulsar/client/__init__.py b/lib/pulsar/client/__init__.py index 86366d3cf7e..4f84a01bde8 100644 --- a/lib/pulsar/client/__init__.py +++ b/lib/pulsar/client/__init__.py @@ -34,7 +34,7 @@ line to ``galaxy.ini``:: test_tool = pulsar://http://remotehost:8913 Remember this must be added after the ``[galaxy:tool_runners]`` header -in the ``universe.ini`` file. +in the ``galaxy.ini`` file. """ diff --git a/lib/pulsar/client/amqp_exchange.py b/lib/pulsar/client/amqp_exchange.py index fd12ace19c6..20591c2259d 100644 --- a/lib/pulsar/client/amqp_exchange.py +++ b/lib/pulsar/client/amqp_exchange.py @@ -85,18 +85,27 @@ class PulsarExchange(object): if heartbeat_thread: heartbeat_thread.join() sleep(DEFAULT_RECONNECT_CONSUMER_WAIT) + except BaseException: + log.exception("Problem consuming queue, consumer quitting in problematic fashion!") + raise def heartbeat(self, connection): log.debug('AMQP heartbeat thread alive') - while connection.connected: - connection.heartbeat_check() - sleep(DEFAULT_HEARTBEAT_WAIT) + try: + while connection.connected: + connection.heartbeat_check() + sleep(DEFAULT_HEARTBEAT_WAIT) + except BaseException: + log.exception("Problem with heartbeat, leaving heartbeat method in problematic state!") + raise log.debug('AMQP heartbeat thread exiting') def publish(self, name, payload): + key = self.__queue_name(name) + log.debug("Begin publishing to key %s" % key) with self.connection(self.__url) as connection: with pools.producers[connection].acquire() as producer: - key = self.__queue_name(name) + log.debug("Have producer for publishing to key %s" % key) producer.publish( payload, serializer='json', @@ -105,6 +114,7 @@ class PulsarExchange(object): routing_key=key, **self.__publish_kwds ) + log.debug("Published to key %s" % key) def __publish_errback(self, exc, interval): log.error("Connection error while publishing: %r", exc, exc_info=1) diff --git a/lib/pulsar/client/transport/__init__.py b/lib/pulsar/client/transport/__init__.py index 1f1925e3c1c..e9d10d3f2c5 100644 --- a/lib/pulsar/client/transport/__init__.py +++ b/lib/pulsar/client/transport/__init__.py @@ -23,9 +23,16 @@ def __get_transport_type(transport_type, os_module): transport_type = 'curl' return transport_type -# TODO: Provide urllib implementation if these unavailable, -# also explore a requests+poster option. -from .curl import get_file -from .curl import post_file +from .curl import curl_available +from .requests import requests_multipart_post_available +if curl_available: + from .curl import get_file + from .curl import post_file +elif requests_multipart_post_available: + from .requests import get_file + from .requests import post_file +else: + from .poster import get_file + from .poster import post_file __all__ = [get_transport, get_file, post_file] diff --git a/lib/pulsar/client/transport/curl.py b/lib/pulsar/client/transport/curl.py index 9594c719541..e4d3d36b624 100644 --- a/lib/pulsar/client/transport/curl.py +++ b/lib/pulsar/client/transport/curl.py @@ -2,10 +2,11 @@ try: from cStringIO import StringIO except ImportError: from io import StringIO +curl_available = True try: from pycurl import Curl except ImportError: - pass + curl_available = False from os.path import getsize diff --git a/lib/pulsar/client/transport/poster.py b/lib/pulsar/client/transport/poster.py new file mode 100644 index 00000000000..a79cd8dc57e --- /dev/null +++ b/lib/pulsar/client/transport/poster.py @@ -0,0 +1,56 @@ +from __future__ import absolute_import +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen +try: + from urllib2 import Request +except ImportError: + from urllib.request import Request +try: + from galaxy import eggs + eggs.require("poster") +except ImportError: + pass + +try: + import poster +except ImportError: + poster = None + +POSTER_UNAVAILABLE_MESSAGE = "Pulsar configured to use poster module - but it is unavailable. Please install poster." + +import logging +log = logging.getLogger(__name__) + + +if poster is not None: + poster.streaminghttp.register_openers() + + +def post_file(url, path): + __ensure_poster() + try: + datagen, headers = poster.encode.multipart_encode({"file": open(path, "rb")}) + request = Request(url, datagen, headers) + return urlopen(request).read() + except: + log.exception("problem") + raise + + +def get_file(url, path): + __ensure_poster() + request = Request(url=url) + response = urlopen(request) + with open(path, 'wb') as output: + while True: + buffer = response.read(1024) + if not buffer: + break + output.write(buffer) + + +def __ensure_poster(): + if poster is None: + raise ImportError(POSTER_UNAVAILABLE_MESSAGE) diff --git a/lib/pulsar/client/transport/requests.py b/lib/pulsar/client/transport/requests.py new file mode 100644 index 00000000000..5c7715192c6 --- /dev/null +++ b/lib/pulsar/client/transport/requests.py @@ -0,0 +1,50 @@ +from __future__ import absolute_import +try: + from galaxy import eggs + eggs.require("requets") +except ImportError: + pass + +try: + import requests +except ImportError: + requests = None +requests_multipart_post_available = False +try: + import requests_toolbelt + requests_multipart_post_available = True +except ImportError: + requests_toolbelt = None + + +REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is unavailable. Please install requests." +REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt module - but it is unavailable. Please install requests_toolbelt." + +import logging +log = logging.getLogger(__name__) + + +def post_file(url, path): + if requests_toolbelt is None: + raise ImportError(REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE) + + __ensure_requests() + m = requests_toolbelt.MultipartEncoder( + fields={'file': ('filename', open(path, 'rb'))} + ) + requests.post(url, data=m, headers={'Content-Type': m.content_type}) + + +def get_file(url, path): + __ensure_requests() + r = requests.get(url, stream=True) + with open(path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + f.flush() + + +def __ensure_requests(): + if requests is None: + raise ImportError(REQUESTS_UNAVAILABLE_MESSAGE) From b376ed766473092ba4328b5782d0fb9217b0294f Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 16 Sep 2014 17:13:17 -0400 Subject: [PATCH 07/11] Fix a bug with config/galaxy.ini in run.sh --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index b05c5401d09..12b384be6af 100755 --- a/run.sh +++ b/run.sh @@ -22,7 +22,7 @@ if [ -z "$GALAXY_CONFIG_FILE" ]; then if [ -f universe_wsgi.ini ]; then GALAXY_CONFIG_FILE=universe_wsgi.ini elif [ -f config/galaxy.ini ]; then - GALAXY_CONFIG_FILE=universe_wsgi.ini + GALAXY_CONFIG_FILE=config/galaxy.ini else GALAXY_CONFIG_FILE=config/galaxy.ini.sample fi From 6e0e5f8042003717a264e100de1dfb2cd2c00662 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 16 Sep 2014 17:51:40 -0400 Subject: [PATCH 08/11] Stop recovering jobs that don't need to be recovered - jobs in the 'NEW' state when tracking in the database. --- lib/galaxy/jobs/handler.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 0f49fa19844..74d333a7b31 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -101,19 +101,22 @@ class JobHandlerQueue( object ): In case the activation is enforced it will filter out the jobs of inactive users. """ jobs_at_startup = [] + if self.track_jobs_in_database: + in_list = ( model.Job.states.QUEUED, + model.Job.states.RUNNING ) + else: + in_list = ( model.Job.states.NEW, + model.Job.states.QUEUED, + model.Job.states.RUNNING ) if self.app.config.user_activation_on: jobs_at_startup = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ .outerjoin( model.User ) \ - .filter( ( ( model.Job.state == model.Job.states.NEW ) \ - | ( model.Job.state == model.Job.states.RUNNING ) \ - | ( model.Job.state == model.Job.states.QUEUED ) ) \ + .filter( model.Job.state.in_( in_list ) \ & ( model.Job.handler == self.app.config.server_name ) \ & or_( ( model.Job.user_id == None ), ( model.User.active == True ) ) ).all() else: jobs_at_startup = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ - .filter( ( ( model.Job.state == model.Job.states.NEW ) \ - | ( model.Job.state == model.Job.states.RUNNING ) \ - | ( model.Job.state == model.Job.states.QUEUED ) ) \ + .filter( model.Job.state.in_( in_list ) \ & ( model.Job.handler == self.app.config.server_name ) ).all() for job in jobs_at_startup: From 48bb7cc6f76ad480ab277144e5c1631c6fdc3798 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 16 Sep 2014 18:29:12 -0400 Subject: [PATCH 09/11] Fix another instance of the renaming that I broke. --- install_and_test_tool_shed_repositories.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_and_test_tool_shed_repositories.sh b/install_and_test_tool_shed_repositories.sh index 5ec57807757..26ad28f13eb 100644 --- a/install_and_test_tool_shed_repositories.sh +++ b/install_and_test_tool_shed_repositories.sh @@ -33,7 +33,7 @@ if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_URL ] ; then fi if [ -z "$GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF" ] ; then - if grep --quiet $GALAXY_INSTALL_TEST_TOOL_SHED_URL tool_sheds_conf.xml; then + if grep --quiet $GALAXY_INSTALL_TEST_TOOL_SHED_URL config/tool_sheds_conf.xml.sample; then echo "Tool sheds configuration tool_sheds_conf.xml ok, proceeding." else echo "ERROR: Tool sheds configuration tool_sheds_conf.xml does not have an entry for $GALAXY_INSTALL_TEST_TOOL_SHED_URL." From 16ce5c0155e0fb5ef99b1b26d747fea97bf5a93a Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 16 Sep 2014 18:29:23 -0400 Subject: [PATCH 10/11] Fix typo in Pulsar client lib. --- lib/pulsar/client/transport/requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pulsar/client/transport/requests.py b/lib/pulsar/client/transport/requests.py index 5c7715192c6..025015d14ed 100644 --- a/lib/pulsar/client/transport/requests.py +++ b/lib/pulsar/client/transport/requests.py @@ -1,7 +1,7 @@ from __future__ import absolute_import try: from galaxy import eggs - eggs.require("requets") + eggs.require("requests") except ImportError: pass From 6ff787d769e7e0e51f5031e15f5f8879e8c2e9ac Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 16 Sep 2014 21:29:10 -0400 Subject: [PATCH 11/11] More tool functional tests for validation stuff. Test default sanitization in repeat. Basic test of simpler santizer and mapping. --- test/functional/tools/samples_tool_conf.xml | 2 + test/functional/tools/validation_repeat.xml | 56 +++++++++++++++++++ .../functional/tools/validation_sanitizer.xml | 35 ++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 test/functional/tools/validation_repeat.xml create mode 100644 test/functional/tools/validation_sanitizer.xml diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 74113f0c83c..76a074ae9c4 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -22,6 +22,8 @@ + + diff --git a/test/functional/tools/validation_repeat.xml b/test/functional/tools/validation_repeat.xml new file mode 100644 index 00000000000..4390b1441df --- /dev/null +++ b/test/functional/tools/validation_repeat.xml @@ -0,0 +1,56 @@ + + + #for $r in $r1 + echo "${r.text}" >> out1; + #end for + #for $r in $r2 + echo "${r.text}" >> out2; + #end for + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/validation_sanitizer.xml b/test/functional/tools/validation_sanitizer.xml new file mode 100644 index 00000000000..e8a47a82108 --- /dev/null +++ b/test/functional/tools/validation_sanitizer.xml @@ -0,0 +1,35 @@ + + + echo "${text}" >> out1; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +