From c99c7b4d842b844677673472b9aa2b13e4123aeb Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 21 Mar 2018 02:51:53 +0000 Subject: [PATCH 1/2] Fix type of some options in config_schema.yml Also: - drop unused `nginx_x_archive_files_base` option, last use removed in commit fe5983aafca1fae0b60ed16838799b58f4aca11c - rebuild config samples and docs. --- config/galaxy.yml.sample | 15 ++-- config/reports.yml.sample | 61 +++++++++++--- config/tool_shed.yml.sample | 90 +++++++++++++++------ doc/source/admin/galaxy_options.rst | 38 +++------ doc/source/admin/reports_options.rst | 28 +++---- lib/galaxy/config.py | 1 - lib/galaxy/webapps/galaxy/config_schema.yml | 34 +++----- 7 files changed, 161 insertions(+), 106 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 17437bbd466..b6ec88c5865 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -752,12 +752,7 @@ galaxy: # Redirect. This should be set to the path defined in the nginx # config as an internal redirect with access to Galaxy's data files # (see documentation linked above). - #nginx_x_accel_redirect_base: false - - # nginx can make use of mod_zip to create zip files containing - # multiple library files. If using X-Accel-Redirect, this can be the - # same value as that option. - #nginx_x_archive_files_base: false + #nginx_x_accel_redirect_base: null # If using compression in the upstream proxy server, use this option # to disable gzipping of library .tar.gz and .zip archives, since the @@ -779,24 +774,24 @@ galaxy: # in detail in the documentation linked above. The upload store is a # temporary directory in which files uploaded by the upload module # will be placed. - #nginx_upload_store: false + #nginx_upload_store: null # This value overrides the action set on the file upload form, e.g. # the web path where the nginx_upload_module has been configured to # intercept upload requests. - #nginx_upload_path: false + #nginx_upload_path: null # Galaxy can also use nginx_upload_module to receive files staged out # upon job completion by remote job runners (i.e. Pulsar) that # initiate staging operations on the remote end. See the Galaxy nginx # documentation for the corresponding nginx configuration. - #nginx_upload_job_files_store: false + #nginx_upload_job_files_store: null # Galaxy can also use nginx_upload_module to receive files staged out # upon job completion by remote job runners (i.e. Pulsar) that # initiate staging operations on the remote end. See the Galaxy nginx # documentation for the corresponding nginx configuration. - #nginx_upload_job_files_path: false + #nginx_upload_job_files_path: null # Have Galaxy manage dynamic proxy component for routing requests to # other services based on Galaxy's session cookie. It will attempt to diff --git a/config/reports.yml.sample b/config/reports.yml.sample index 2fa53484616..438d9b14b46 100644 --- a/config/reports.yml.sample +++ b/config/reports.yml.sample @@ -2,20 +2,59 @@ uwsgi: # The address and port on which to listen. By default, only listen to # localhost (reports will not be accessible over the network). Use - # '0.0.0.0' to listen on all available network interfaces. + # ':9001' to listen on all available network interfaces. http: 127.0.0.1:9001 - threads: 8 + # Number of web server (worker) processes to fork after the + # application has loaded. + processes: 1 - http-raw-body: True + # Number of threads for each web server process. + threads: 4 - offload-threads: 8 + # Number of threads for serving static content and handling internal + # routing requests. + offload-threads: 2 + # Mapping to serve style content. static-map: /static/style=static/style/blue + + # Mapping to serve the remainder of the static content. static-map: /static=static + # Enable the master process manager. Disabled by default for maximum + # compatibility with CTRL+C, but should be enabled for use with + # --daemon and/or production deployments. + master: false + + # Path to the application's Python virtual environment. + virtualenv: .venv + + # Path to the application's Python library. + pythonpath: lib + + # The entry point which returns the web application (e.g. Galaxy, + # Reports, etc.) that you are loading. module: galaxy.webapps.reports.buildapp:uwsgi_app() + # Cause uWSGI to respect the traditional behavior of dying on SIGTERM + # (its default is to brutally reload workers) + die-on-term: true + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGINT (its default is to brutally kill workers) + hook-master-start: unix_signal:2 gracefully_kill_them_all + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGTERM (its default is to brutally kill workers) + hook-master-start: unix_signal:15 gracefully_kill_them_all + + # Feature necessary for proper mule signal handling + py-call-osafterfork: true + + # Ensure application threads will run if `threads` is unset. + enable-threads: true + reports: # If running behind a proxy server and Galaxy is served from a @@ -28,7 +67,7 @@ reports: # path as the prefix in the filter above. This value becomes the # "path" attribute set in the cookie so the cookies from each instance # will not clobber each other. - #cookie_path: None + #cookie_path: null # Verbosity of console log messages. Acceptable values can be found # here: https://docs.python.org/2/library/logging.html#logging-levels @@ -38,7 +77,7 @@ reports: # Galaxy instances, so sqlite (and the default value below) is not # supported. An SQLAlchemy connection string should be used specify an # external database. - #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE + #database_connection: 'sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE' # Where dataset files are saved Temporary storage for additional # datasets, this should be shared through the cluster @@ -53,20 +92,20 @@ reports: #template_cache_path: database/compiled_templates/reports # Configuration for debugging middleware - #debug: False + #debug: false # Check for WSGI compliance. - #use_lint: False + #use_lint: false # NEVER enable this on a public site (even test or QA) - #use_interactive: True + #use_interactive: true # Write thread status periodically to 'heartbeat.log' (careful, uses # disk space rapidly!) - #use_heartbeat: True + #use_heartbeat: true # Profiling middleware (cProfile based) - #use_profile: True + #use_profile: true # Mail #smtp_server: yourserver@yourfacility.edu diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample index c76956580ef..78f9a466812 100644 --- a/config/tool_shed.yml.sample +++ b/config/tool_shed.yml.sample @@ -2,20 +2,59 @@ uwsgi: # The address and port on which to listen. By default, only listen to # localhost (tool_shed will not be accessible over the network). Use - # '0.0.0.0' to listen on all available network interfaces. + # ':9009' to listen on all available network interfaces. http: 127.0.0.1:9009 - threads: 8 + # Number of web server (worker) processes to fork after the + # application has loaded. + processes: 1 - http-raw-body: True + # Number of threads for each web server process. + threads: 4 - offload-threads: 8 + # Number of threads for serving static content and handling internal + # routing requests. + offload-threads: 2 + # Mapping to serve style content. static-map: /static/style=static/style/blue + + # Mapping to serve the remainder of the static content. static-map: /static=static + # Enable the master process manager. Disabled by default for maximum + # compatibility with CTRL+C, but should be enabled for use with + # --daemon and/or production deployments. + master: false + + # Path to the application's Python virtual environment. + virtualenv: .venv + + # Path to the application's Python library. + pythonpath: lib + + # The entry point which returns the web application (e.g. Galaxy, + # Reports, etc.) that you are loading. module: galaxy.webapps.tool_shed.buildapp:uwsgi_app() + # Cause uWSGI to respect the traditional behavior of dying on SIGTERM + # (its default is to brutally reload workers) + die-on-term: true + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGINT (its default is to brutally kill workers) + hook-master-start: unix_signal:2 gracefully_kill_them_all + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGTERM (its default is to brutally kill workers) + hook-master-start: unix_signal:15 gracefully_kill_them_all + + # Feature necessary for proper mule signal handling + py-call-osafterfork: true + + # Ensure application threads will run if `threads` is unset. + enable-threads: true + tool_shed: # Verbosity of console log messages. Acceptable values can be found @@ -27,11 +66,11 @@ tool_shed: # string to specify an external database instead. This string takes # many options which are explained in detail in the config file # documentation. - #database_connection: sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE + #database_connection: 'sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE' # Where the hgweb.config file is stored. The default is the Galaxy # installation directory. - #hgweb_config_dir: None + #hgweb_config_dir: null # Where tool shed repositories are stored. #file_path: database/community_files @@ -57,7 +96,7 @@ tool_shed: # allow full text API searching over the repositories and tools within # the Tool Shed given that you specify the following two config # options. - #toolshed_search_on: True + #toolshed_search_on: true # -- Repository and Tool search Using the script located at # scripts/build_ts_whoosh_index.py you can generate search index and @@ -98,7 +137,7 @@ tool_shed: # You can enter tracking code here to track visitor's behavior through # your Google Analytics account. Example: UA-XXXXXXXX-Y - #ga_code: None + #ga_code: null # The Tool Shed encodes various internal values when these values will # be output in some format (for example, in a URL or cookie). You @@ -115,7 +154,7 @@ tool_shed: # header in the request. Enabling remote user disables regular logins. # For more information, see: # https://wiki.galaxyproject.org/Admin/Config/ApacheProxy - #use_remote_user: False + #use_remote_user: false # If use_remote_user is enabled, anyone who can log in to the Galaxy # host may impersonate any other user by simply sending the @@ -126,22 +165,25 @@ tool_shed: #remote_user_secret: changethisinproductiontoo # Configuration for debugging middleware - #debug: False + #debug: false # Check for WSGI compliance. - #use_lint: False + #use_lint: false + + # Intercept print statements and show them on the returned page. + #use_printdebug: true # NEVER enable this on a public site (even test or QA) - #use_interactive: True + #use_interactive: true # Administrative users - set this to a comma-separated list of valid # Tool Shed users (email addresses). These users will have access to # the Admin section of the server, and will have access to create # users, groups, roles, libraries, and more. - #admin_users: None + #admin_users: null # Force everyone to log in (disable anonymous access) - #require_login: False + #require_login: false # For use by email messages sent from the tool shed #smtp_server: smtp.your_tool_shed_server @@ -152,44 +194,44 @@ tool_shed: # If your SMTP server requires a username and password, you can # provide them here (password in cleartext here, but if your server # supports STARTTLS it will be sent over the network encrypted). - #smtp_username: None + #smtp_username: null # If your SMTP server requires a username and password, you can # provide them here (password in cleartext here, but if your server # supports STARTTLS it will be sent over the network encrypted). - #smtp_password: None + #smtp_password: null # If your SMTP server requires SSL from the beginning of the # connection - #smtp_ssl: False + #smtp_ssl: false # The URL linked by the "Support" link in the "Help" menu. - #support_url: https://wiki.galaxyproject.org/Support + #support_url: 'https://wiki.galaxyproject.org/Support' # Address to join mailing list #mailing_join_addr: galaxy-announce-join@bx.psu.edu # Write thread status periodically to 'heartbeat.log' (careful, uses # disk space rapidly!) - #use_heartbeat: True + #use_heartbeat: true # Profiling middleware (cProfile based) - #use_profile: True + #use_profile: true # Enable creation of Galaxy flavor Docker Image - #enable_galaxy_flavor_docker_image: False + #enable_galaxy_flavor_docker_image: false # Show a message box under the masthead. - #message_box_visible: False + #message_box_visible: false # Show a message box under the masthead. - #message_box_content: None + #message_box_content: null # Show a message box under the masthead. #message_box_class: info # Serving static files (needed if running standalone) - #static_enabled: True + #static_enabled: true # Serving static files (needed if running standalone) #static_cache_time: 360 diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 6275dd5a5e0..d518ce4c2da 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -71,7 +71,7 @@ gone away", you will want to set this to some positive value (7200 should work). :Default: ``-1`` -:Type: str +:Type: int ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -429,7 +429,7 @@ scheme that may work in wider range of scenarios than the watchdog default. :Default: ``false`` -:Type: bool +:Type: str ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -594,7 +594,7 @@ which will use a less efficient monitoring scheme that may work in wider range of scenarios than the watchdog default. :Default: ``false`` -:Type: bool +:Type: str ~~~~~~~~~~~~~~~~~~~~ @@ -1517,20 +1517,8 @@ Redirect. This should be set to the path defined in the nginx config as an internal redirect with access to Galaxy's data files (see documentation linked above). -:Default: ``false`` -:Type: bool - - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``nginx_x_archive_files_base`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:Description: - nginx can make use of mod_zip to create zip files containing - multiple library files. If using X-Accel-Redirect, this can be - the same value as that option. -:Default: ``false`` -:Type: bool +:Default: ``None`` +:Type: str ~~~~~~~~~~~~~~~~~ @@ -1572,8 +1560,8 @@ explained in detail in the documentation linked above. The upload store is a temporary directory in which files uploaded by the upload module will be placed. -:Default: ``false`` -:Type: bool +:Default: ``None`` +:Type: str ~~~~~~~~~~~~~~~~~~~~~ @@ -1584,8 +1572,8 @@ This value overrides the action set on the file upload form, e.g. the web path where the nginx_upload_module has been configured to intercept upload requests. -:Default: ``false`` -:Type: bool +:Default: ``None`` +:Type: str ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1597,8 +1585,8 @@ out upon job completion by remote job runners (i.e. Pulsar) that initiate staging operations on the remote end. See the Galaxy nginx documentation for the corresponding nginx configuration. -:Default: ``false`` -:Type: bool +:Default: ``None`` +:Type: str ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1610,8 +1598,8 @@ out upon job completion by remote job runners (i.e. Pulsar) that initiate staging operations on the remote end. See the Galaxy nginx documentation for the corresponding nginx configuration. -:Default: ``false`` -:Type: bool +:Default: ``None`` +:Type: str ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/admin/reports_options.rst b/doc/source/admin/reports_options.rst index 90557f7c844..a2fd8882cfb 100644 --- a/doc/source/admin/reports_options.rst +++ b/doc/source/admin/reports_options.rst @@ -6,7 +6,7 @@ If running behind a proxy server and Galaxy is served from a subdirectory, enable the proxy-prefix filter and set the prefix in the [filter:proxy-prefix] section above. -:Default: proxy-prefix +:Default: ``proxy-prefix`` :Type: str @@ -20,7 +20,7 @@ same path as the prefix in the filter above. This value becomes the "path" attribute set in the cookie so the cookies from each instance will not clobber each other. -:Default: None +:Default: ``None`` :Type: str @@ -32,7 +32,7 @@ Verbosity of console log messages. Acceptable values can be found here: https://docs.python.org/2/library/logging.html#logging- levels -:Default: DEBUG +:Default: ``DEBUG`` :Type: str @@ -45,7 +45,7 @@ Galaxy instances, so sqlite (and the default value below) is not supported. An SQLAlchemy connection string should be used specify an external database. -:Default: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE +:Default: ``sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE`` :Type: str @@ -56,7 +56,7 @@ :Description: Where dataset files are saved Temporary storage for additional datasets, this should be shared through the cluster -:Default: database/files +:Default: ``database/files`` :Type: str @@ -67,7 +67,7 @@ :Description: Where dataset files are saved Temporary storage for additional datasets, this should be shared through the cluster -:Default: database/tmp +:Default: ``database/tmp`` :Type: str @@ -78,7 +78,7 @@ :Description: Mako templates are compiled as needed and cached for reuse, this directory is used for the cache -:Default: database/compiled_templates/reports +:Default: ``database/compiled_templates/reports`` :Type: str @@ -88,7 +88,7 @@ :Description: Configuration for debugging middleware -:Default: False +:Default: ``false`` :Type: bool @@ -98,7 +98,7 @@ :Description: Check for WSGI compliance. -:Default: False +:Default: ``false`` :Type: bool @@ -108,7 +108,7 @@ :Description: NEVER enable this on a public site (even test or QA) -:Default: True +:Default: ``true`` :Type: bool @@ -119,7 +119,7 @@ :Description: Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!) -:Default: True +:Default: ``true`` :Type: bool @@ -129,7 +129,7 @@ :Description: Profiling middleware (cProfile based) -:Default: True +:Default: ``true`` :Type: bool @@ -139,7 +139,7 @@ :Description: Mail -:Default: yourserver@yourfacility.edu +:Default: ``yourserver@yourfacility.edu`` :Type: str @@ -149,7 +149,7 @@ :Description: Mail -:Default: your_bugs@bx.psu.edu +:Default: ``your_bugs@bx.psu.edu`` :Type: str diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index ed4ebc1c7c1..b2e3e5893fa 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -498,7 +498,6 @@ class Configuration(object): self.upstream_gzip = string_as_bool(kwargs.get('upstream_gzip', False)) self.apache_xsendfile = string_as_bool(kwargs.get('apache_xsendfile', False)) self.nginx_x_accel_redirect_base = kwargs.get('nginx_x_accel_redirect_base', False) - self.nginx_x_archive_files_base = kwargs.get('nginx_x_archive_files_base', False) self.nginx_upload_store = kwargs.get('nginx_upload_store', False) self.nginx_upload_path = kwargs.get('nginx_upload_path', False) self.nginx_upload_job_files_store = kwargs.get('nginx_upload_job_files_store', False) diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index e858a5cb1d9..edb3fd1720f 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -82,7 +82,7 @@ mapping: processes. database_engine_option_pool_recycle: - type: str + type: int default: -1 required: false desc: | @@ -331,7 +331,7 @@ mapping: install from in the admin interface (.sample used if default does not exist). watch_tools: - type: bool + type: str default: false required: false desc: | @@ -454,7 +454,7 @@ mapping: when installed from a ToolShed. Defaults to tool_data_path. watch_tool_data_dir: - type: bool + type: str default: false required: false desc: | @@ -1139,22 +1139,14 @@ mapping: this to True to inform Galaxy that mod_xsendfile is enabled upstream. nginx_x_accel_redirect_base: - type: bool - default: false + type: str + default: null required: false desc: | The same download handling can be done by nginx using X-Accel-Redirect. This should be set to the path defined in the nginx config as an internal redirect with access to Galaxy's data files (see documentation linked above). - nginx_x_archive_files_base: - type: bool - default: false - required: false - desc: | - nginx can make use of mod_zip to create zip files containing multiple library - files. If using X-Accel-Redirect, this can be the same value as that option. - upstream_gzip: type: bool default: false @@ -1179,8 +1171,8 @@ mapping: `X-Frame-Options` header. nginx_upload_store: - type: bool - default: false + type: str + default: null required: false desc: | nginx can also handle file uploads (user-to-Galaxy) via nginx_upload_module. @@ -1189,8 +1181,8 @@ mapping: which files uploaded by the upload module will be placed. nginx_upload_path: - type: bool - default: false + type: str + default: null required: false desc: | This value overrides the action set on the file upload form, e.g. the web @@ -1198,8 +1190,8 @@ mapping: requests. nginx_upload_job_files_store: - type: bool - default: false + type: str + default: null required: false desc: | Galaxy can also use nginx_upload_module to receive files staged out upon job @@ -1208,8 +1200,8 @@ mapping: corresponding nginx configuration. nginx_upload_job_files_path: - type: bool - default: false + type: str + default: null required: false desc: | Galaxy can also use nginx_upload_module to receive files staged out upon job From 80ac55b3ff132cf6e1778b99cb6ffb01bec9cfe8 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 21 Mar 2018 03:01:57 +0000 Subject: [PATCH 2/2] Don't quote ints in .ini to .yml config conversion --- lib/galaxy/webapps/config_manage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 714d4628544..44ae449e910 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -751,10 +751,11 @@ def _parse_option_value(option_value): if isinstance(option_value, OptionValue): option = option_value.option value = option_value.value - option = option_value.option # Hack to get nicer YAML values during conversion if option.get("type", "str") == "bool": value = str(value).lower() == "true" + elif option.get("type", "str") == "int": + value = int(value) else: value = option_value option = OPTION_DEFAULTS