From 29c78e984e52ac08b1b36a03e39b65c6edbe6200 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 8 Mar 2016 08:42:15 +0000 Subject: [PATCH 1/7] Fix virtualenv broken with #1847. --- run_reports.sh | 8 ++++++++ run_tool_shed.sh | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/run_reports.sh b/run_reports.sh index dc24e77f5c5..874988ad254 100755 --- a/run_reports.sh +++ b/run_reports.sh @@ -11,6 +11,14 @@ cd `dirname $0` +: ${GALAXY_VIRTUAL_ENV:=.venv} + +if [ -d "$GALAXY_VIRTUAL_ENV" ]; +then + printf "Activating virtualenv at $GALAXY_VIRTUAL_ENV\n" + . "$GALAXY_VIRTUAL_ENV/bin/activate" +fi + ./scripts/common_startup.sh --skip-samples if [ -z "$GALAXY_REPORTS_CONFIG" ]; then diff --git a/run_tool_shed.sh b/run_tool_shed.sh index 9f6a0b3eaed..ae6169d60db 100755 --- a/run_tool_shed.sh +++ b/run_tool_shed.sh @@ -2,6 +2,14 @@ cd `dirname $0` +: ${GALAXY_VIRTUAL_ENV:=.venv} + +if [ -d "$GALAXY_VIRTUAL_ENV" ]; +then + printf "Activating virtualenv at $GALAXY_VIRTUAL_ENV\n" + . "$GALAXY_VIRTUAL_ENV/bin/activate" +fi + ./scripts/common_startup.sh tool_shed=`./lib/tool_shed/scripts/bootstrap_tool_shed/parse_run_sh_args.sh $@` From 5824bc04af9270afbebcb2c8f9b6b3bea371b788 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 8 Mar 2016 12:40:27 +0000 Subject: [PATCH 2/7] Fix duplicate print statement reported by @dannon... https://github.com/galaxyproject/galaxy/pull/1890#issuecomment-193873380 --- run_reports.sh | 1 - run_tool_shed.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/run_reports.sh b/run_reports.sh index 874988ad254..6699df1589b 100755 --- a/run_reports.sh +++ b/run_reports.sh @@ -15,7 +15,6 @@ cd `dirname $0` if [ -d "$GALAXY_VIRTUAL_ENV" ]; then - printf "Activating virtualenv at $GALAXY_VIRTUAL_ENV\n" . "$GALAXY_VIRTUAL_ENV/bin/activate" fi diff --git a/run_tool_shed.sh b/run_tool_shed.sh index ae6169d60db..e1cb19f9bb1 100755 --- a/run_tool_shed.sh +++ b/run_tool_shed.sh @@ -6,7 +6,6 @@ cd `dirname $0` if [ -d "$GALAXY_VIRTUAL_ENV" ]; then - printf "Activating virtualenv at $GALAXY_VIRTUAL_ENV\n" . "$GALAXY_VIRTUAL_ENV/bin/activate" fi From 438661e8945c1bb317f75dabe3db429ea779255d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 10 Mar 2016 14:34:00 +0000 Subject: [PATCH 3/7] Another fix for virtualenv fixes. --- run_reports.sh | 4 ++-- run_tool_shed.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/run_reports.sh b/run_reports.sh index 6699df1589b..eb2de06ad97 100755 --- a/run_reports.sh +++ b/run_reports.sh @@ -11,6 +11,8 @@ cd `dirname $0` +./scripts/common_startup.sh --skip-samples + : ${GALAXY_VIRTUAL_ENV:=.venv} if [ -d "$GALAXY_VIRTUAL_ENV" ]; @@ -18,8 +20,6 @@ then . "$GALAXY_VIRTUAL_ENV/bin/activate" fi -./scripts/common_startup.sh --skip-samples - if [ -z "$GALAXY_REPORTS_CONFIG" ]; then if [ -f reports_wsgi.ini ]; then GALAXY_REPORTS_CONFIG=reports_wsgi.ini diff --git a/run_tool_shed.sh b/run_tool_shed.sh index e1cb19f9bb1..e34002fcc32 100755 --- a/run_tool_shed.sh +++ b/run_tool_shed.sh @@ -2,6 +2,8 @@ cd `dirname $0` +./scripts/common_startup.sh + : ${GALAXY_VIRTUAL_ENV:=.venv} if [ -d "$GALAXY_VIRTUAL_ENV" ]; @@ -9,7 +11,6 @@ then . "$GALAXY_VIRTUAL_ENV/bin/activate" fi -./scripts/common_startup.sh tool_shed=`./lib/tool_shed/scripts/bootstrap_tool_shed/parse_run_sh_args.sh $@` args=$@ From f277f99aed411fede7d1cb91d3177469922a58ae Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 17 Mar 2016 10:41:36 -0400 Subject: [PATCH 4/7] Fix handling of time in fluent logger. --- lib/galaxy/util/log/fluent_log.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/util/log/fluent_log.py b/lib/galaxy/util/log/fluent_log.py index e1e6224e74a..8e0752e9808 100644 --- a/lib/galaxy/util/log/fluent_log.py +++ b/lib/galaxy/util/log/fluent_log.py @@ -3,6 +3,7 @@ Provides a `TraceLogger` implementation that logs to a fluentd collector """ import threading +import time try: from fluent.sender import FluentSender @@ -34,11 +35,10 @@ class FluentTraceLogger( object ): del self.thread_local.context[key] self.lock.release() - def log( self, label, time=None, **kwargs ): + def log( self, label, event_time=None, **kwargs ): self.lock.acquire() if hasattr( self.thread_local, 'context' ): kwargs.update( self.thread_local.context ) self.lock.release() - if time is None: - time = int( time.time() ) - self.sender.emit_with_time( label, time, kwargs ) + event_time = event_time or time.time() + self.sender.emit_with_time( label, int(event_time), kwargs ) From 87bcb5f5a1b6801e948b2cc0f5dc676c7faca682 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 17 Mar 2016 12:09:02 -0400 Subject: [PATCH 5/7] Fix in a different way not requiring signature change; still not thrilled about overlapping kwarg names w/ stdlib module names. --- lib/galaxy/util/log/fluent_log.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/util/log/fluent_log.py b/lib/galaxy/util/log/fluent_log.py index 8e0752e9808..704482a2e76 100644 --- a/lib/galaxy/util/log/fluent_log.py +++ b/lib/galaxy/util/log/fluent_log.py @@ -3,7 +3,7 @@ Provides a `TraceLogger` implementation that logs to a fluentd collector """ import threading -import time +from time import time as now try: from fluent.sender import FluentSender @@ -35,10 +35,10 @@ class FluentTraceLogger( object ): del self.thread_local.context[key] self.lock.release() - def log( self, label, event_time=None, **kwargs ): + def log( self, label, time=None, **kwargs ): self.lock.acquire() if hasattr( self.thread_local, 'context' ): kwargs.update( self.thread_local.context ) self.lock.release() - event_time = event_time or time.time() - self.sender.emit_with_time( label, int(event_time), kwargs ) + time = time or now() + self.sender.emit_with_time( label, int(time), kwargs ) From c8e48bfaccb5d10552df4200e616a9eee49206b0 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 17 Mar 2016 13:03:04 -0400 Subject: [PATCH 6/7] Add fluentd configuration to galaxy.ini.sample --- config/galaxy.ini.sample | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index 1e98a97621e..4e052277cf1 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -657,6 +657,13 @@ nglims_config_file = tool-data/nglims.yaml # log_events and log_actions functionality will eventually be merged. #log_actions = True + +# Fluentd configuration. Various events can be logged to the fluentd instance +# configured below by enabling fluent_log. +#fluent_log = False +#fluent_host = localhost +#fluent_port = 24224 + # Sanitize all HTML tool output. By default, all tool output served as # 'text/html' will be sanitized thoroughly. This can be disabled if you have # special tools that require unaltered output. WARNING: disabling this does From 98c65dcee08338ea80a7e34251f8fa4ed6bd8dd5 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 17 Mar 2016 13:13:19 -0400 Subject: [PATCH 7/7] Swap back to the first fix, update metrics api to use new signature. --- lib/galaxy/util/log/fluent_log.py | 8 ++++---- lib/galaxy/webapps/galaxy/api/metrics.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/util/log/fluent_log.py b/lib/galaxy/util/log/fluent_log.py index 704482a2e76..8e0752e9808 100644 --- a/lib/galaxy/util/log/fluent_log.py +++ b/lib/galaxy/util/log/fluent_log.py @@ -3,7 +3,7 @@ Provides a `TraceLogger` implementation that logs to a fluentd collector """ import threading -from time import time as now +import time try: from fluent.sender import FluentSender @@ -35,10 +35,10 @@ class FluentTraceLogger( object ): del self.thread_local.context[key] self.lock.release() - def log( self, label, time=None, **kwargs ): + def log( self, label, event_time=None, **kwargs ): self.lock.acquire() if hasattr( self.thread_local, 'context' ): kwargs.update( self.thread_local.context ) self.lock.release() - time = time or now() - self.sender.emit_with_time( label, int(time), kwargs ) + event_time = event_time or time.time() + self.sender.emit_with_time( label, int(event_time), kwargs ) diff --git a/lib/galaxy/webapps/galaxy/api/metrics.py b/lib/galaxy/webapps/galaxy/api/metrics.py index cba0c244fb6..dd6e727339b 100644 --- a/lib/galaxy/webapps/galaxy/api/metrics.py +++ b/lib/galaxy/webapps/galaxy/api/metrics.py @@ -87,7 +87,7 @@ class MetricsController( BaseAPIController ): """ if trans.app.trace_logger: for label, time, kwargs in metrics: - trans.app.trace_logger.log( label, time=int( time ), **kwargs ) + trans.app.trace_logger.log( label, event_time=int( time ), **kwargs ) elif self.debugging: for label, time, kwargs in metrics: log.debug( '%s %s %s', label, time, kwargs )