Merge remote-tracking branch 'upstream/release_22.05' into dev

doc/source/admin/config_logging.rst

@@ -45,6 +45,20 @@ To change the log file name or location, use the ``$GALAXY_LOG`` environment var
 
     $ GALAXY_LOG=/path/to/galaxy/logfile sh run.sh --daemon
 
+It is also possible to specify the path to the log file using the ``log_destination`` configuration option in
+``galaxy.yml``. Additionally, it is possible to automatically rotate logs once the log file reaches a given size, using
+the ``log_rotate_size`` and ``log_rotate_count`` options, which control the size at which the log is rotated, and the
+number of rotated logs to keep, respectively:
+
+.. code-block:: yaml
+
+    galaxy:
+        # Set log file path
+        log_destination: /srv/galaxy/log/galaxy.log
+        # Rotate once log reaches 100 MB
+        log_rotate_size: 100 MB
+        # Keep the 10 most recent log files
+        log_rotate_count: 10
 
 Advanced Configuration
 ----------------------------

doc/source/admin/galaxy_options.rst

@@ -2563,12 +2563,55 @@
 :Type: bool
 
 
+~~~~~~~~~~~~~~~~~~~
+``log_destination``
+~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Log destination, defaults to special value "stdout" that logs to
+    standard output. If set to anything else, then it will be
+    interpreted as a path that will be used as the log file, and
+    logging to stdout will be disabled.
+:Default: ``stdout``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~
+``log_rotate_size``
+~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Size of log file at which size it will be rotated as per the
+    documentation in
+    https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler
+    If log_rotate_count is not also set, no log rotation will be
+    performed. A value of 0 (the default) means no rotation. Size can
+    be a number of bytes or a human-friendly representation like "100
+    MB" or "1G".
+:Default: ``0``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~
+``log_rotate_count``
+~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Number of log file backups to keep, per the documentation in
+    https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler
+    Any additional rotated log files will automatically be pruned. If
+    log_rotate_size is not also set, no log rotation will be
+    performed. A value of 0 (the default) means no rotation.
+:Default: ``0``
+:Type: int
+
+
 ~~~~~~~~~~~~~
 ``log_level``
 ~~~~~~~~~~~~~
 
 :Description:
-    Verbosity of console log messages.  Acceptable values can be found
+    Verbosity of console log messages. Acceptable values can be found
     here: https://docs.python.org/library/logging.html#logging-levels
     A custom debug level of "TRACE" is available for even more
     verbosity.
@@ -2581,9 +2624,8 @@
 ~~~~~~~~~~~
 
 :Description:
-    Controls where and how the server logs messages. If unset, the
-    default is to log all messages to standard output at the level
-    defined by the `log_level` configuration option. Configuration is
+    Controls where and how the server logs messages. If set, overrides
+    all settings in the log_* configuration options. Configuration is
     described in the documentation at:
     https://docs.galaxyproject.org/en/master/admin/config_logging.html
 :Default: ``None``

lib/galaxy/config/__init__.py

@@ -36,6 +36,7 @@ from galaxy.config.schema import AppSchema
 from galaxy.exceptions import ConfigurationError
 from galaxy.util import (
     listify,
+    size_to_bytes,
     string_as_bool,
     unicodify,
 )
@@ -103,6 +104,12 @@ LOGGING_CONFIG_DEFAULT: Dict[str, Any] = {
             "level": "INFO",
             "qualname": "botocore",
         },
+        "gunicorn.access": {
+            "level": "INFO",
+            "qualname": "gunicorn.access",
+            "propagate": False,
+            "handlers": ["console"],
+        },
     },
     "filters": {
         "stack": {
@@ -1104,6 +1111,8 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
             }
 
         log_destination = kwargs.get("log_destination")
+        log_rotate_size = size_to_bytes(unicodify(kwargs.get("log_rotate_size", 0)))
+        log_rotate_count = int(kwargs.get("log_rotate_count", 0))
         galaxy_daemon_log_destination = os.environ.get("GALAXY_DAEMON_LOG")
         if log_destination == "stdout":
             LOGGING_CONFIG_DEFAULT["handlers"]["console"] = {
@@ -1115,19 +1124,23 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
             }
         elif log_destination:
             LOGGING_CONFIG_DEFAULT["handlers"]["console"] = {
-                "class": "logging.FileHandler",
+                "class": "logging.handlers.RotatingFileHandler",
                 "formatter": "stack",
                 "level": "DEBUG",
                 "filename": log_destination,
                 "filters": ["stack"],
+                "maxBytes": log_rotate_size,
+                "backupCount": log_rotate_count,
             }
         if galaxy_daemon_log_destination:
             LOGGING_CONFIG_DEFAULT["handlers"]["files"] = {
-                "class": "logging.FileHandler",
+                "class": "logging.handlers.RotatingFileHandler",
                 "formatter": "stack",
                 "level": "DEBUG",
                 "filename": galaxy_daemon_log_destination,
                 "filters": ["stack"],
+                "maxBytes": log_rotate_size,
+                "backupCount": log_rotate_count,
             }
             LOGGING_CONFIG_DEFAULT["root"]["handlers"].append("files")
 

lib/galaxy/config/sample/galaxy.yml.sample

@@ -1354,14 +1354,35 @@ galaxy:
   # "loggers" section does not appear in this configuration file.
   #auto_configure_logging: true
 
-  # Verbosity of console log messages.  Acceptable values can be found
+  # Log destination, defaults to special value "stdout" that logs to
+  # standard output. If set to anything else, then it will be
+  # interpreted as a path that will be used as the log file, and logging
+  # to stdout will be disabled.
+  #log_destination: stdout
+
+  # Size of log file at which size it will be rotated as per the
+  # documentation in
+  # https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler
+  # If log_rotate_count is not also set, no log rotation will be
+  # performed. A value of 0 (the default) means no rotation. Size can be
+  # a number of bytes or a human-friendly representation like "100 MB"
+  # or "1G".
+  #log_rotate_size: '0'
+
+  # Number of log file backups to keep, per the documentation in
+  # https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler
+  # Any additional rotated log files will automatically be pruned. If
+  # log_rotate_size is not also set, no log rotation will be performed.
+  # A value of 0 (the default) means no rotation.
+  #log_rotate_count: 0
+
+  # Verbosity of console log messages. Acceptable values can be found
   # here: https://docs.python.org/library/logging.html#logging-levels A
   # custom debug level of "TRACE" is available for even more verbosity.
   #log_level: DEBUG
 
-  # Controls where and how the server logs messages. If unset, the
-  # default is to log all messages to standard output at the level
-  # defined by the `log_level` configuration option. Configuration is
+  # Controls where and how the server logs messages. If set, overrides
+  # all settings in the log_* configuration options. Configuration is
   # described in the documentation at:
   # https://docs.galaxyproject.org/en/master/admin/config_logging.html
   #logging: null

lib/galaxy/config/schemas/config_schema.yml

@@ -1862,12 +1862,41 @@ mapping:
           If true, Galaxy will attempt to configure a simple root logger if a
           "loggers" section does not appear in this configuration file.
 
+      log_destination:
+        type: str
+        default: stdout
+        required: false
+        desc: |
+          Log destination, defaults to special value "stdout" that logs to standard output. If set to anything else,
+          then it will be interpreted as a path that will be used as the log file, and logging to stdout will be
+          disabled.
+
+      log_rotate_size:
+        type: str
+        default: "0"
+        required: false
+        desc: |
+          Size of log file at which size it will be rotated as per the documentation in
+          https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler
+          If log_rotate_count is not also set, no log rotation will be performed. A value of 0 (the default) means no
+          rotation. Size can be a number of bytes or a human-friendly representation like "100 MB" or "1G".
+
+      log_rotate_count:
+        type: int
+        default: 0
+        required: false
+        desc: |
+          Number of log file backups to keep, per the documentation in
+          https://docs.python.org/library/logging.handlers.html#logging.handlers.RotatingFileHandler
+          Any additional rotated log files will automatically be pruned. If log_rotate_size is not also set, no log
+          rotation will be performed. A value of 0 (the default) means no rotation.
+
       log_level:
         type: str
         default: DEBUG
         required: false
         desc: |
-          Verbosity of console log messages.  Acceptable values can be found here:
+          Verbosity of console log messages. Acceptable values can be found here:
           https://docs.python.org/library/logging.html#logging-levels
           A custom debug level of "TRACE" is available for even more verbosity.
 
@@ -1875,9 +1904,8 @@ mapping:
         type: map
         allowempty: true
         desc: |
-          Controls where and how the server logs messages. If unset, the default is to log all messages to standard
-          output at the level defined by the `log_level` configuration option. Configuration is described in the
-          documentation at:
+          Controls where and how the server logs messages. If set, overrides all settings in the log_* configuration
+          options. Configuration is described in the documentation at:
           https://docs.galaxyproject.org/en/master/admin/config_logging.html
 
       database_engine_option_echo:

lib/galaxy/dependencies/dev-requirements.txt

@@ -73,7 +73,7 @@ fs==2.4.16
 funcsigs==1.0.2
 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
 galaxy-sequence-utils==1.1.5
-gravity==0.13.2; python_version >= "3.6"
+gravity==0.13.3; python_version >= "3.6"
 greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0"
 gunicorn==20.1.0; python_version >= "3.5"
 gxformat2==0.15.0

lib/galaxy/dependencies/pinned-requirements.txt

@@ -61,7 +61,7 @@ fs==2.4.16
 funcsigs==1.0.2
 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
 galaxy-sequence-utils==1.1.5
-gravity==0.13.2; python_version >= "3.6"
+gravity==0.13.3; python_version >= "3.6"
 greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0"
 gunicorn==20.1.0; python_version >= "3.5"
 gxformat2==0.15.0

lib/galaxy/schema/schema.py

@@ -42,7 +42,10 @@ from galaxy.schema.fields import (
     EncodedDatabaseIdField,
     ModelClassField,
 )
-from galaxy.schema.types import RelativeUrl
+from galaxy.schema.types import (
+    OffsetNaiveDatetime,
+    RelativeUrl,
+)
 
 USER_MODEL_CLASS_NAME = "User"
 GROUP_MODEL_CLASS_NAME = "Group"
@@ -1117,8 +1120,8 @@ class JobIndexQueryPayload(Model):
     user_id: Optional[DecodedDatabaseIdField] = None
     tool_ids: Optional[List[str]] = None
     tool_ids_like: Optional[List[str]] = None
-    date_range_min: Optional[Union[datetime, date]] = None
-    date_range_max: Optional[Union[datetime, date]] = None
+    date_range_min: Optional[Union[OffsetNaiveDatetime, date]] = None
+    date_range_max: Optional[Union[OffsetNaiveDatetime, date]] = None
     history_id: Optional[DecodedDatabaseIdField] = None
     workflow_id: Optional[DecodedDatabaseIdField] = None
     invocation_id: Optional[DecodedDatabaseIdField] = None

lib/galaxy/schema/types.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+
+from pydantic.datetime_parse import parse_datetime
 from typing_extensions import Literal
 
 # Relative URLs cannot be validated with AnyUrl, they need a scheme.
@@ -5,3 +8,14 @@ from typing_extensions import Literal
 RelativeUrl = str
 
 LatestLiteral = Literal["latest"]
+
+
+class OffsetNaiveDatetime(datetime):
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, v):
+        v = parse_datetime(v)
+        return v.replace(tzinfo=None) - v.utcoffset() if v.tzinfo else v

lib/galaxy/tool_util/deps/singularity_util.py

@@ -75,10 +75,10 @@ def build_singularity_run_command(
         sudo=sudo,
         sudo_cmd=sudo_cmd,
     )
-    if cleanenv:
-        command_parts.append("--cleanenv")
     command_parts.append("-s")
     command_parts.append("exec")
+    if cleanenv:
+        command_parts.append("--cleanenv")
     for volume in volumes:
         command_parts.extend(["-B", str(volume)])
     if home is not None:

lib/galaxy/webapps/galaxy/api/jobs.py

@@ -38,6 +38,7 @@ from galaxy.managers.jobs import (
 )
 from galaxy.schema.fields import EncodedDatabaseIdField
 from galaxy.schema.schema import JobIndexSortByEnum
+from galaxy.schema.types import OffsetNaiveDatetime
 from galaxy.util import listify
 from galaxy.web import (
     expose_api,
@@ -103,13 +104,13 @@ ToolIdLikeQueryParam: Optional[str] = Query(
     description="Limit listing of jobs to those that match one of the included tool ID sql-like patterns. If none, all are returned",
 )
 
-DateRangeMinQueryParam: Optional[Union[datetime, date]] = Query(
+DateRangeMinQueryParam: Optional[Union[OffsetNaiveDatetime, date]] = Query(
     default=None,
     title="Date Range Minimum",
     description="Limit listing of jobs to those that are updated after specified date (e.g. '2014-01-01')",
 )
 
-DateRangeMaxQueryParam: Optional[Union[datetime, date]] = Query(
+DateRangeMaxQueryParam: Optional[Union[OffsetNaiveDatetime, date]] = Query(
     default=None,
     title="Date Range Maximum",
     description="Limit listing of jobs to those that are updated before specified date (e.g. '2014-01-01')",

lib/galaxy/webapps/galaxy/buildapp.py

@@ -1369,7 +1369,7 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf):
     # Error middleware
     app = wrap_if_allowed(app, stack, ErrorMiddleware, args=(conf,))
     # Transaction logging (apache access.log style)
-    if asbool(conf.get("use_translogger", True)):
+    if asbool(conf.get("use_translogger", False)):
         from galaxy.web.framework.middleware.translogger import TransLogger
 
         app = wrap_if_allowed(app, stack, TransLogger)

test/unit/tool_util/test_singularity_util.py

@@ -0,0 +1,18 @@
+from galaxy.tool_util.deps.singularity_util import build_singularity_run_command
+
+
+def test_build_singularity_run_command_defaults():
+    cmd = build_singularity_run_command(
+        container_command="echo hi",
+        image="busybox",
+    )
+    assert cmd == "singularity -s exec --cleanenv busybox echo hi"
+
+
+def test_build_singularity_run_command_no_cleanenv():
+    cmd = build_singularity_run_command(
+        container_command="echo hi",
+        image="busybox",
+        cleanenv=False,
+    )
+    assert cmd == "singularity -s exec busybox echo hi"

test/unit/webapps/api/test_datetime_parsing.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+
+from galaxy.schema.types import OffsetNaiveDatetime
+
+
+class Time(BaseModel):
+    time: OffsetNaiveDatetime
+
+
+def test_naive_datetime_parsing():
+    with_tz = Time(time="2022-08-15T11:29:32.853974+02:00")
+    without_tz = Time(time="2022-08-15T09:29:32.853974")
+    assert with_tz.time == without_tz.time
+    assert with_tz.time.tzinfo is None