diff --git a/doc/source/admin/config_logging.rst b/doc/source/admin/config_logging.rst index faeba11ddc5..cd6e4460a9a 100644 --- a/doc/source/admin/config_logging.rst +++ b/doc/source/admin/config_logging.rst @@ -45,6 +45,20 @@ To change the log file name or location, use the ``$GALAXY_LOG`` environment var $ GALAXY_LOG=/path/to/galaxy/logfile sh run.sh --daemon +It is also possible to specify the path to the log file using the ``log_destination`` configuration option in +``galaxy.yml``. Additionally, it is possible to automatically rotate logs once the log file reaches a given size, using +the ``log_rotate_size`` and ``log_rotate_count`` options, which control the size at which the log is rotated, and the +number of rotated logs to keep, respectively: + +.. code-block:: yaml + + galaxy: + # Set log file path + log_destination: /srv/galaxy/log/galaxy.log + # Rotate once log reaches 100 MB + log_rotate_size: 100 MB + # Keep the 10 most recent log files + log_rotate_count: 10 Advanced Configuration ---------------------------- diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index ea70b56f314..6ac89a7a9ed 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -2563,12 +2563,55 @@ :Type: bool +~~~~~~~~~~~~~~~~~~~ +``log_destination`` +~~~~~~~~~~~~~~~~~~~ + +:Description: + Log destination, defaults to special value "stdout" that logs to + standard output. If set to anything else, then it will be + interpreted as a path that will be used as the log file, and + logging to stdout will be disabled. +:Default: ``stdout`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~ +``log_rotate_size`` +~~~~~~~~~~~~~~~~~~~ + +:Description: + Size of log file at which size it will be rotated as per the + documentation in + https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler + If log_rotate_count is not also set, no log rotation will be + performed. A value of 0 (the default) means no rotation. Size can + be a number of bytes or a human-friendly representation like "100 + MB" or "1G". +:Default: ``0`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~ +``log_rotate_count`` +~~~~~~~~~~~~~~~~~~~~ + +:Description: + Number of log file backups to keep, per the documentation in + https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler + Any additional rotated log files will automatically be pruned. If + log_rotate_size is not also set, no log rotation will be + performed. A value of 0 (the default) means no rotation. +:Default: ``0`` +:Type: int + + ~~~~~~~~~~~~~ ``log_level`` ~~~~~~~~~~~~~ :Description: - Verbosity of console log messages. Acceptable values can be found + Verbosity of console log messages. Acceptable values can be found here: https://docs.python.org/library/logging.html#logging-levels A custom debug level of "TRACE" is available for even more verbosity. @@ -2581,9 +2624,8 @@ ~~~~~~~~~~~ :Description: - Controls where and how the server logs messages. If unset, the - default is to log all messages to standard output at the level - defined by the `log_level` configuration option. Configuration is + Controls where and how the server logs messages. If set, overrides + all settings in the log_* configuration options. Configuration is described in the documentation at: https://docs.galaxyproject.org/en/master/admin/config_logging.html :Default: ``None`` diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index b6a81f55c56..579c41a7fab 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -36,6 +36,7 @@ from galaxy.config.schema import AppSchema from galaxy.exceptions import ConfigurationError from galaxy.util import ( listify, + size_to_bytes, string_as_bool, unicodify, ) @@ -103,6 +104,12 @@ LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { "level": "INFO", "qualname": "botocore", }, + "gunicorn.access": { + "level": "INFO", + "qualname": "gunicorn.access", + "propagate": False, + "handlers": ["console"], + }, }, "filters": { "stack": { @@ -1104,6 +1111,8 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): } log_destination = kwargs.get("log_destination") + log_rotate_size = size_to_bytes(unicodify(kwargs.get("log_rotate_size", 0))) + log_rotate_count = int(kwargs.get("log_rotate_count", 0)) galaxy_daemon_log_destination = os.environ.get("GALAXY_DAEMON_LOG") if log_destination == "stdout": LOGGING_CONFIG_DEFAULT["handlers"]["console"] = { @@ -1115,19 +1124,23 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): } elif log_destination: LOGGING_CONFIG_DEFAULT["handlers"]["console"] = { - "class": "logging.FileHandler", + "class": "logging.handlers.RotatingFileHandler", "formatter": "stack", "level": "DEBUG", "filename": log_destination, "filters": ["stack"], + "maxBytes": log_rotate_size, + "backupCount": log_rotate_count, } if galaxy_daemon_log_destination: LOGGING_CONFIG_DEFAULT["handlers"]["files"] = { - "class": "logging.FileHandler", + "class": "logging.handlers.RotatingFileHandler", "formatter": "stack", "level": "DEBUG", "filename": galaxy_daemon_log_destination, "filters": ["stack"], + "maxBytes": log_rotate_size, + "backupCount": log_rotate_count, } LOGGING_CONFIG_DEFAULT["root"]["handlers"].append("files") diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 42a2046c874..16e3655caba 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1354,14 +1354,35 @@ galaxy: # "loggers" section does not appear in this configuration file. #auto_configure_logging: true - # Verbosity of console log messages. Acceptable values can be found + # Log destination, defaults to special value "stdout" that logs to + # standard output. If set to anything else, then it will be + # interpreted as a path that will be used as the log file, and logging + # to stdout will be disabled. + #log_destination: stdout + + # Size of log file at which size it will be rotated as per the + # documentation in + # https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler + # If log_rotate_count is not also set, no log rotation will be + # performed. A value of 0 (the default) means no rotation. Size can be + # a number of bytes or a human-friendly representation like "100 MB" + # or "1G". + #log_rotate_size: '0' + + # Number of log file backups to keep, per the documentation in + # https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler + # Any additional rotated log files will automatically be pruned. If + # log_rotate_size is not also set, no log rotation will be performed. + # A value of 0 (the default) means no rotation. + #log_rotate_count: 0 + + # Verbosity of console log messages. Acceptable values can be found # here: https://docs.python.org/library/logging.html#logging-levels A # custom debug level of "TRACE" is available for even more verbosity. #log_level: DEBUG - # Controls where and how the server logs messages. If unset, the - # default is to log all messages to standard output at the level - # defined by the `log_level` configuration option. Configuration is + # Controls where and how the server logs messages. If set, overrides + # all settings in the log_* configuration options. Configuration is # described in the documentation at: # https://docs.galaxyproject.org/en/master/admin/config_logging.html #logging: null diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 1d7633ffb8e..ac411b40dae 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -1862,12 +1862,41 @@ mapping: If true, Galaxy will attempt to configure a simple root logger if a "loggers" section does not appear in this configuration file. + log_destination: + type: str + default: stdout + required: false + desc: | + Log destination, defaults to special value "stdout" that logs to standard output. If set to anything else, + then it will be interpreted as a path that will be used as the log file, and logging to stdout will be + disabled. + + log_rotate_size: + type: str + default: "0" + required: false + desc: | + Size of log file at which size it will be rotated as per the documentation in + https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler + If log_rotate_count is not also set, no log rotation will be performed. A value of 0 (the default) means no + rotation. Size can be a number of bytes or a human-friendly representation like "100 MB" or "1G". + + log_rotate_count: + type: int + default: 0 + required: false + desc: | + Number of log file backups to keep, per the documentation in + https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler + Any additional rotated log files will automatically be pruned. If log_rotate_size is not also set, no log + rotation will be performed. A value of 0 (the default) means no rotation. + log_level: type: str default: DEBUG required: false desc: | - Verbosity of console log messages. Acceptable values can be found here: + Verbosity of console log messages. Acceptable values can be found here: https://docs.python.org/library/logging.html#logging-levels A custom debug level of "TRACE" is available for even more verbosity. @@ -1875,9 +1904,8 @@ mapping: type: map allowempty: true desc: | - Controls where and how the server logs messages. If unset, the default is to log all messages to standard - output at the level defined by the `log_level` configuration option. Configuration is described in the - documentation at: + Controls where and how the server logs messages. If set, overrides all settings in the log_* configuration + options. Configuration is described in the documentation at: https://docs.galaxyproject.org/en/master/admin/config_logging.html database_engine_option_echo: diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 7ab6040dc39..6f5be35b08b 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -73,7 +73,7 @@ fs==2.4.16 funcsigs==1.0.2 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") galaxy-sequence-utils==1.1.5 -gravity==0.13.2; python_version >= "3.6" +gravity==0.13.3; python_version >= "3.6" greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0" gunicorn==20.1.0; python_version >= "3.5" gxformat2==0.15.0 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index dfd5f19b7bc..725cb7a90b7 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -61,7 +61,7 @@ fs==2.4.16 funcsigs==1.0.2 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") galaxy-sequence-utils==1.1.5 -gravity==0.13.2; python_version >= "3.6" +gravity==0.13.3; python_version >= "3.6" greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0" gunicorn==20.1.0; python_version >= "3.5" gxformat2==0.15.0 diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 770c36312b4..0beb9cde9a2 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -42,7 +42,10 @@ from galaxy.schema.fields import ( EncodedDatabaseIdField, ModelClassField, ) -from galaxy.schema.types import RelativeUrl +from galaxy.schema.types import ( + OffsetNaiveDatetime, + RelativeUrl, +) USER_MODEL_CLASS_NAME = "User" GROUP_MODEL_CLASS_NAME = "Group" @@ -1117,8 +1120,8 @@ class JobIndexQueryPayload(Model): user_id: Optional[DecodedDatabaseIdField] = None tool_ids: Optional[List[str]] = None tool_ids_like: Optional[List[str]] = None - date_range_min: Optional[Union[datetime, date]] = None - date_range_max: Optional[Union[datetime, date]] = None + date_range_min: Optional[Union[OffsetNaiveDatetime, date]] = None + date_range_max: Optional[Union[OffsetNaiveDatetime, date]] = None history_id: Optional[DecodedDatabaseIdField] = None workflow_id: Optional[DecodedDatabaseIdField] = None invocation_id: Optional[DecodedDatabaseIdField] = None diff --git a/lib/galaxy/schema/types.py b/lib/galaxy/schema/types.py index cf39c391d3a..49b42126907 100644 --- a/lib/galaxy/schema/types.py +++ b/lib/galaxy/schema/types.py @@ -1,3 +1,6 @@ +from datetime import datetime + +from pydantic.datetime_parse import parse_datetime from typing_extensions import Literal # Relative URLs cannot be validated with AnyUrl, they need a scheme. @@ -5,3 +8,14 @@ from typing_extensions import Literal RelativeUrl = str LatestLiteral = Literal["latest"] + + +class OffsetNaiveDatetime(datetime): + @classmethod + def __get_validators__(cls): + yield cls.validate + + @classmethod + def validate(cls, v): + v = parse_datetime(v) + return v.replace(tzinfo=None) - v.utcoffset() if v.tzinfo else v diff --git a/lib/galaxy/tool_util/deps/singularity_util.py b/lib/galaxy/tool_util/deps/singularity_util.py index 33d8559cd33..0a97f052210 100644 --- a/lib/galaxy/tool_util/deps/singularity_util.py +++ b/lib/galaxy/tool_util/deps/singularity_util.py @@ -75,10 +75,10 @@ def build_singularity_run_command( sudo=sudo, sudo_cmd=sudo_cmd, ) - if cleanenv: - command_parts.append("--cleanenv") command_parts.append("-s") command_parts.append("exec") + if cleanenv: + command_parts.append("--cleanenv") for volume in volumes: command_parts.extend(["-B", str(volume)]) if home is not None: diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index d88ba5c40f8..1eb13167b58 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -38,6 +38,7 @@ from galaxy.managers.jobs import ( ) from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.schema.schema import JobIndexSortByEnum +from galaxy.schema.types import OffsetNaiveDatetime from galaxy.util import listify from galaxy.web import ( expose_api, @@ -103,13 +104,13 @@ ToolIdLikeQueryParam: Optional[str] = Query( description="Limit listing of jobs to those that match one of the included tool ID sql-like patterns. If none, all are returned", ) -DateRangeMinQueryParam: Optional[Union[datetime, date]] = Query( +DateRangeMinQueryParam: Optional[Union[OffsetNaiveDatetime, date]] = Query( default=None, title="Date Range Minimum", description="Limit listing of jobs to those that are updated after specified date (e.g. '2014-01-01')", ) -DateRangeMaxQueryParam: Optional[Union[datetime, date]] = Query( +DateRangeMaxQueryParam: Optional[Union[OffsetNaiveDatetime, date]] = Query( default=None, title="Date Range Maximum", description="Limit listing of jobs to those that are updated before specified date (e.g. '2014-01-01')", diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 83757e53074..465ad801b0e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -1369,7 +1369,7 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): # Error middleware app = wrap_if_allowed(app, stack, ErrorMiddleware, args=(conf,)) # Transaction logging (apache access.log style) - if asbool(conf.get("use_translogger", True)): + if asbool(conf.get("use_translogger", False)): from galaxy.web.framework.middleware.translogger import TransLogger app = wrap_if_allowed(app, stack, TransLogger) diff --git a/test/unit/tool_util/test_singularity_util.py b/test/unit/tool_util/test_singularity_util.py new file mode 100644 index 00000000000..8a02b2c97ee --- /dev/null +++ b/test/unit/tool_util/test_singularity_util.py @@ -0,0 +1,18 @@ +from galaxy.tool_util.deps.singularity_util import build_singularity_run_command + + +def test_build_singularity_run_command_defaults(): + cmd = build_singularity_run_command( + container_command="echo hi", + image="busybox", + ) + assert cmd == "singularity -s exec --cleanenv busybox echo hi" + + +def test_build_singularity_run_command_no_cleanenv(): + cmd = build_singularity_run_command( + container_command="echo hi", + image="busybox", + cleanenv=False, + ) + assert cmd == "singularity -s exec busybox echo hi" diff --git a/test/unit/webapps/api/test_datetime_parsing.py b/test/unit/webapps/api/test_datetime_parsing.py new file mode 100644 index 00000000000..29ea26b14ff --- /dev/null +++ b/test/unit/webapps/api/test_datetime_parsing.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel + +from galaxy.schema.types import OffsetNaiveDatetime + + +class Time(BaseModel): + time: OffsetNaiveDatetime + + +def test_naive_datetime_parsing(): + with_tz = Time(time="2022-08-15T11:29:32.853974+02:00") + without_tz = Time(time="2022-08-15T09:29:32.853974") + assert with_tz.time == without_tz.time + assert with_tz.time.tzinfo is None