Merge pull request #3179 from jmchilton/web_config_overhaul_0

Configuration & Server Overhaul - Tool Shed and Reports

Makefile

@@ -10,6 +10,7 @@ MY_UPSTREAM:=origin
 VENV?=.venv
 # Source virtualenv to execute command (flake8, sphinx, twine, etc...)
 IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi;
+CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/webapps/config_manage.py
 PROJECT_URL?=https://github.com/galaxyproject/galaxy
 GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01
 GRUNT_EXEC?=node_modules/grunt-cli/bin/grunt
@@ -52,6 +53,42 @@ open-project: ## open project on github
 lint: ## check style using tox and flake8 for Python 2 and Python 3
 	$(IN_VENV) tox -e py27-lint && tox -e py34-lint
 
+uwsgi-rebuild-validation: ## rebuild uwsgi_config.yml kwalify schema against latest uwsgi master.
+	$(CONFIG_MANAGE) build_uwsgi_yaml
+
+tool-shed-config-validate: ## validate tool shed YAML configuration file
+	$(CONFIG_MANAGE) validate tool_shed
+
+tool-shed-config-lint: ## lint tool shed YAML configuration file
+	$(CONFIG_MANAGE) lint tool_shed
+
+tool-shed-config-convert-dry-run: ## convert old style tool shed ini to yaml (dry run)
+	$(CONFIG_MANAGE) convert tool_shed --dry-run
+
+tool-shed-config-convert: ## convert old style tool shed ini to yaml
+	$(CONFIG_MANAGE) convert tool_shed
+
+tool-shed-config-rebuild-sample: ## Rebuild sample tool shed yaml file from schema
+	$(CONFIG_MANAGE) build_sample_yaml tool_shed --add-comments
+
+reports-config-validate: ## validate reports YAML configuration file
+	$(CONFIG_MANAGE) validate reports
+
+reports-config-convert-dry-run: ## convert old style reports ini to yaml (dry run)
+	$(CONFIG_MANAGE) convert reports --dry-run
+
+reports-config-convert: ## convert old style reports ini to yaml
+	$(CONFIG_MANAGE) convert reports
+
+reports-config-rebuild-sample: ## Rebuild sample reports yaml file from schema
+	$(CONFIG_MANAGE) build_sample_yaml reports --add-comments
+
+reports-config-lint: ## lint reports YAML configuration file
+	$(CONFIG_MANAGE) lint reports
+
+reports-config-rebuild-rst: ## Rebuild sample reports RST docs
+	$(CONFIG_MANAGE) build_rst reports > doc/source/admin/reports_options.rst
+
 release-ensure-upstream: ## Ensure upstream branch for release commands setup
 ifeq (shell git remote -v | grep $(RELEASE_UPSTREAM), )
 	git remote add $(RELEASE_UPSTREAM) git@github.com:galaxyproject/galaxy.git

config/reports.yml.sample

@@ -0,0 +1,73 @@
+uwsgi:
+
+  # The address and port on which to listen.  By default, only listen to
+  # localhost (reports will not be accessible over the network).  Use
+  # '0.0.0.0' to listen on all available network interfaces.
+  http: 127.0.0.1:9001
+
+  threads: 8
+
+  http-raw-body: True
+
+  offload-threads: 8
+
+  module: galaxy.webapps.reports.buildapp:uwsgi_app()
+
+reports:
+
+  # If running behind a proxy server and Galaxy is served from a
+  # subdirectory, enable the proxy-prefix filter and set the prefix in
+  # the [filter:proxy-prefix] section above.
+  #filter-with: proxy-prefix
+
+  # If proxy-prefix is enabled and you're running more than one Galaxy
+  # instance behind one hostname, you will want to set this to the same
+  # path as the prefix in the filter above.  This value becomes the
+  # "path" attribute set in the cookie so the cookies from each instance
+  # will not clobber each other.
+  #cookie_path: None
+
+  # Verbosity of console log messages.  Acceptable values can be found
+  # here: https://docs.python.org/2/library/logging.html#logging-levels
+  #log_level: DEBUG
+
+  # Database connection Galaxy reports are intended for production
+  # Galaxy instances, so sqlite (and the default value below) is not
+  # supported. An SQLAlchemy connection string should be used specify an
+  # external database.
+  #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
+
+  # Where dataset files are saved Temporary storage for additional
+  # datasets, this should be shared through the cluster
+  #file_path: database/files
+
+  # Where dataset files are saved Temporary storage for additional
+  # datasets, this should be shared through the cluster
+  #new_file_path: database/tmp
+
+  # Mako templates are compiled as needed and cached for reuse, this
+  # directory is used for the cache
+  #template_cache_path: database/compiled_templates/reports
+
+  # Configuration for debugging middleware
+  #debug: False
+
+  # Check for WSGI compliance.
+  #use_lint: False
+
+  # NEVER enable this on a public site (even test or QA)
+  #use_interactive: True
+
+  # Write thread status periodically to 'heartbeat.log' (careful, uses
+  # disk space rapidly!)
+  #use_heartbeat: True
+
+  # Profiling middleware (cProfile based)
+  #use_profile: True
+
+  # Mail
+  #smtp_server: yourserver@yourfacility.edu
+
+  # Mail
+  #error_email_to: your_bugs@bx.psu.edu
+

config/tool_shed.yml.sample

@@ -0,0 +1,208 @@
+uwsgi:
+
+  # The address and port on which to listen.  By default, only listen to
+  # localhost (tool_shed will not be accessible over the network).  Use
+  # '0.0.0.0' to listen on all available network interfaces.
+  http: 127.0.0.1:9009
+
+  threads: 8
+
+  http-raw-body: True
+
+  offload-threads: 8
+
+  module: galaxy.webapps.tool_shed.buildapp:uwsgi_app()
+
+tool_shed:
+
+  # Verbosity of console log messages.  Acceptable values can be found
+  # here: https://docs.python.org/2/library/logging.html#logging-levels
+  #log_level: DEBUG
+
+  # By default, the Tool Shed uses a SQLite database at
+  # 'database/community.sqlite'.  You may use a SQLAlchemy connection
+  # string to specify an external database instead.  This string takes
+  # many options which are explained in detail in the config file
+  # documentation.
+  #database_connection: sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE
+
+  # Where the hgweb.config file is stored. The default is the Galaxy
+  # installation directory.
+  #hgweb_config_dir: None
+
+  # Where tool shed repositories are stored.
+  #file_path: database/community_files
+
+  # Temporary storage for additional datasets, this should be shared
+  # through the cluster
+  #new_file_path: database/tmp
+
+  # File containing old-style genome builds
+  #builds_file_path: tool-data/shared/ucsc/builds.txt
+
+  # Format string used when showing date and time information. The
+  # string may contain: - the directives used by Python time.strftime()
+  # function (see
+  # https://docs.python.org/2/library/time.html#time.strftime ), -
+  # $locale (complete format string for the server locale), - $iso8601
+  # (complete format string as specified by ISO 8601 international
+  # standard).
+  #pretty_datetime_format: $locale (UTC)
+
+  # -- Repository and Tool search Using the script located at
+  # scripts/build_ts_whoosh_index.py you can generate search index and
+  # allow full text API searching over the repositories and tools within
+  # the Tool Shed given that you specify the following two config
+  # options.
+  #toolshed_search_on: True
+
+  # -- Repository and Tool search Using the script located at
+  # scripts/build_ts_whoosh_index.py you can generate search index and
+  # allow full text API searching over the repositories and tools within
+  # the Tool Shed given that you specify the following two config
+  # options.
+  #whoosh_index_dir: database/toolshed_whoosh_indexes
+
+  # For searching repositories at /api/repositories:
+  #repo_name_boost: 0.9
+
+  # For searching repositories at /api/repositories:
+  #repo_description_boost: 0.6
+
+  # For searching repositories at /api/repositories:
+  #repo_long_description_boost: 0.5
+
+  # For searching repositories at /api/repositories:
+  #repo_homepage_url_boost: 0.3
+
+  # For searching repositories at /api/repositories:
+  #repo_remote_repository_url_boost: 0.2
+
+  # For searching repositories at /api/repositories:
+  #repo_owner_username_boost: 0.3
+
+  # For searching tools at /api/tools
+  #tool_name_boost: 1.2
+
+  # For searching tools at /api/tools
+  #tool_description_boost: 0.6
+
+  # For searching tools at /api/tools
+  #tool_help_boost: 0.4
+
+  # For searching tools at /api/tools
+  #tool_repo_owner_username: 0.3
+
+  # You can enter tracking code here to track visitor's behavior through
+  # your Google Analytics account. Example: UA-XXXXXXXX-Y
+  #ga_code: None
+
+  # The Tool Shed encodes various internal values when these values will
+  # be output in some format (for example, in a URL or cookie).  You
+  # should set a key to be used by the algorithm that encodes and
+  # decodes these values.  It can be any string.  If left unchanged,
+  # anyone could construct a cookie that would grant them access to
+  # others' sessions. One simple way to generate a value for this is
+  # with the shell command:   python -c 'import time; print time.time()'
+  # | md5sum | cut -f 1 -d ' '
+  #id_secret: changethisinproductiontoo
+
+  # User authentication can be delegated to an upstream proxy server
+  # (usually Apache).  The upstream proxy should set a REMOTE_USER
+  # header in the request. Enabling remote user disables regular logins.
+  # For more information, see:
+  # https://wiki.galaxyproject.org/Admin/Config/ApacheProxy
+  #use_remote_user: False
+
+  # If use_remote_user is enabled, anyone who can log in to the Galaxy
+  # host may impersonate any other user by simply sending the
+  # appropriate header. Thus a secret shared between the upstream proxy
+  # server, and Galaxy is required. If anyone other than the Galaxy user
+  # is using the server, then apache/nginx should pass a value in the
+  # header 'GX_SECRET' that is identical the one below
+  #remote_user_secret: changethisinproductiontoo
+
+  # Configuration for debugging middleware
+  #debug: False
+
+  # Check for WSGI compliance.
+  #use_lint: False
+
+  # NEVER enable this on a public site (even test or QA)
+  #use_interactive: True
+
+  # Administrative users - set this to a comma-separated list of valid
+  # Tool Shed users (email addresses).  These users will have access to
+  # the Admin section of the server, and will have access to create
+  # users, groups, roles, libraries, and more.
+  #admin_users: None
+
+  # Force everyone to log in (disable anonymous access)
+  #require_login: False
+
+  # For use by email messages sent from the tool shed
+  #smtp_server: smtp.your_tool_shed_server
+
+  # For use by email messages sent from the tool shed
+  #email_from: your_tool_shed_email@server
+
+  # If your SMTP server requires a username and password, you can
+  # provide them here (password in cleartext here, but if your server
+  # supports STARTTLS it will be sent over the network encrypted).
+  #smtp_username: None
+
+  # If your SMTP server requires a username and password, you can
+  # provide them here (password in cleartext here, but if your server
+  # supports STARTTLS it will be sent over the network encrypted).
+  #smtp_password: None
+
+  # If your SMTP server requires SSL from the beginning of the
+  # connection
+  #smtp_ssl: False
+
+  # The URL linked by the "Support" link in the "Help" menu.
+  #support_url: https://wiki.galaxyproject.org/Support
+
+  # Address to join mailing list
+  #mailing_join_addr: galaxy-announce-join@bx.psu.edu
+
+  # Write thread status periodically to 'heartbeat.log' (careful, uses
+  # disk  space rapidly!)
+  #use_heartbeat: True
+
+  # Profiling middleware (cProfile based)
+  #use_profile: True
+
+  # Enable creation of Galaxy flavor Docker Image
+  #enable_galaxy_flavor_docker_image: False
+
+  # Show a message box under the masthead.
+  #message_box_visible: False
+
+  # Show a message box under the masthead.
+  #message_box_content: None
+
+  # Show a message box under the masthead.
+  #message_box_class: info
+
+  # Serving static files (needed if running standalone)
+  #static_enabled: True
+
+  # Serving static files (needed if running standalone)
+  #static_cache_time: 360
+
+  # Serving static files (needed if running standalone)
+  #static_dir: static/
+
+  # Serving static files (needed if running standalone)
+  #static_images_dir: static/images
+
+  # Serving static files (needed if running standalone)
+  #static_favicon_dir: static/favicon.ico
+
+  # Serving static files (needed if running standalone)
+  #static_scripts_dir: static/scripts/
+
+  # Serving static files (needed if running standalone)
+  #static_style_dir: static/style/blue
+

doc/source/admin/reports.rst

@@ -0,0 +1,57 @@
+Galaxy Reports
+========================================
+
+Overview
+----------------------------
+
+For admins wishing to have more information on the usage of a Galaxy instance,
+one can configure and use the Galaxy Reports application. This is a separate
+application that can run beside Galaxy and must target the same database.
+
+The Galaxy Reports application's easy to use web interface will give you information on the 
+following (and more):
+
+- Jobs
+
+    - Today's jobs
+    - Jobs per day this month
+    - Jobs in error per day this month
+    - All unfinished jobs
+    - Jobs per month
+    - Jobs in error per month
+    - Jobs per user
+    - Jobs per tool
+
+- Workflows
+
+    - Workflows per month
+    - Workflows per user
+
+- Users
+
+    - Registered users
+    - Date of last login
+    - User disk usage
+
+- System
+
+    - Disk space maintenance
+
+
+Configuration
+----------------------------
+
+- Configure ``config/reports.yml`` in the same manner as your main galaxy instance (i.e., same database connection, but different port). This is a uwsgi YAML configuration file and should contain a reports section with app-specific configuration (options described below).
+
+    - The default port for the reports application is ``9001``, and like Galaxy it only binds to localhost by default.
+    - ``database_connection`` should match the value used in your Galaxy configuration
+    - ``database_connection`` should point at a Postgres database, experimental support for MySQL is available but sqlite is not supported at all.
+
+- Run reports in a uWSGI server with ``sh run_reports.sh``
+- Use a web browser and go to the address you configured in ``reports.yml`` (defaults to http://localhost:9001/)
+
+----------------------------
+Configuration Options
+----------------------------
+
+.. include:: reports_options.rst

doc/source/admin/reports_options.rst

@@ -0,0 +1,156 @@
+~~~~~~~~~~~~~~~
+``filter-with``
+~~~~~~~~~~~~~~~
+
+:Description:
+    If running behind a proxy server and Galaxy is served from a
+    subdirectory, enable the proxy-prefix filter and set the prefix in
+    the [filter:proxy-prefix] section above.
+:Default: proxy-prefix
+:Type: str
+
+
+~~~~~~~~~~~~~~~
+``cookie_path``
+~~~~~~~~~~~~~~~
+
+:Description:
+    If proxy-prefix is enabled and you're running more than one Galaxy
+    instance behind one hostname, you will want to set this to the
+    same path as the prefix in the filter above.  This value becomes
+    the "path" attribute set in the cookie so the cookies from each
+    instance will not clobber each other.
+:Default: None
+:Type: str
+
+
+~~~~~~~~~~~~~
+``log_level``
+~~~~~~~~~~~~~
+
+:Description:
+    Verbosity of console log messages.  Acceptable values can be found
+    here: https://docs.python.org/2/library/logging.html#logging-
+    levels
+:Default: DEBUG
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~
+``database_connection``
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Database connection Galaxy reports are intended for production
+    Galaxy instances, so sqlite (and the default value below) is not
+    supported. An SQLAlchemy connection string should be used specify
+    an external database.
+:Default: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
+:Type: str
+
+
+~~~~~~~~~~~~~
+``file_path``
+~~~~~~~~~~~~~
+
+:Description:
+    Where dataset files are saved Temporary storage for additional
+    datasets, this should be shared through the cluster
+:Default: database/files
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~
+``new_file_path``
+~~~~~~~~~~~~~~~~~
+
+:Description:
+    Where dataset files are saved Temporary storage for additional
+    datasets, this should be shared through the cluster
+:Default: database/tmp
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~
+``template_cache_path``
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Mako templates are compiled as needed and cached for reuse, this
+    directory is used for the cache
+:Default: database/compiled_templates/reports
+:Type: str
+
+
+~~~~~~~~~
+``debug``
+~~~~~~~~~
+
+:Description:
+    Configuration for debugging middleware
+:Default: False
+:Type: bool
+
+
+~~~~~~~~~~~~
+``use_lint``
+~~~~~~~~~~~~
+
+:Description:
+    Check for WSGI compliance.
+:Default: False
+:Type: bool
+
+
+~~~~~~~~~~~~~~~~~~~
+``use_interactive``
+~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    NEVER enable this on a public site (even test or QA)
+:Default: True
+:Type: bool
+
+
+~~~~~~~~~~~~~~~~~
+``use_heartbeat``
+~~~~~~~~~~~~~~~~~
+
+:Description:
+    Write thread status periodically to 'heartbeat.log' (careful, uses
+    disk space rapidly!)
+:Default: True
+:Type: bool
+
+
+~~~~~~~~~~~~~~~
+``use_profile``
+~~~~~~~~~~~~~~~
+
+:Description:
+    Profiling middleware (cProfile based)
+:Default: True
+:Type: bool
+
+
+~~~~~~~~~~~~~~~
+``smtp_server``
+~~~~~~~~~~~~~~~
+
+:Description:
+    Mail
+:Default: yourserver@yourfacility.edu
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~
+``error_email_to``
+~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Mail
+:Default: your_bugs@bx.psu.edu
+:Type: str
+
+
+

lib/galaxy/dependencies/__init__.py

@@ -22,7 +22,7 @@ class ConditionalDependencies( object ):
         self.get_conditional_requirements()
 
     def parse_configs( self ):
-        self.config = load_app_properties( ini_file=self.config_file )
+        self.config = load_app_properties( config_file=self.config_file )
         job_conf_xml = self.config.get(
             "job_config_file",
             join( dirname( self.config_file ), 'job_conf.xml' ) )

lib/galaxy/dependencies/pinned-requirements.txt

@@ -6,6 +6,7 @@ SQLAlchemy==1.0.15
 mercurial==3.7.3
 numpy==1.9.2
 pycrypto==2.6.1
+uWSGI==2.0.15
 
 # Install python_lzo if you want to support indexed access to lzo-compressed
 # locally cached maf files via bx-python
@@ -32,6 +33,11 @@ Whoosh==2.7.4
 testfixtures==4.10.0
 galaxy_sequence_utils==1.0.2
 
+# pykwalify and dependencies
+pykwalify==1.5.1
+python-dateutil==2.5.3
+docopt==0.6.2
+
 # Cheetah and dependencies
 Cheetah==2.4.4
 Markdown==2.6.3

lib/galaxy/model/orm/scripts.py

@@ -3,6 +3,7 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...).
 """
 import logging
 
+from galaxy.util import listify
 from galaxy.util.properties import find_config_file, load_app_properties
 
 
@@ -16,22 +17,23 @@ DATABASE = {
     "galaxy":
         {
             'repo': 'lib/galaxy/model/migrate',
-            'old_config_file': 'universe_wsgi.ini',
+            'old_config_files': ['universe_wsgi.ini'],
             'default_sqlite_file': './database/universe.sqlite',
             'config_override': 'GALAXY_CONFIG_',
         },
     "tool_shed":
         {
             'repo': 'lib/galaxy/webapps/tool_shed/model/migrate',
-            'config_file': 'config/tool_shed.ini',
-            'old_config_file': 'tool_shed_wsgi.ini',
+            'config_file': 'config/tool_shed.yml',
+            'old_config_files': ['config/tool_shed.ini', 'tool_shed_wsgi.ini'],
             'default_sqlite_file': './database/community.sqlite',
             'config_override': 'TOOL_SHED_CONFIG_',
+            'config_section': 'tool_shed',
         },
     "install":
         {
             'repo': 'lib/galaxy/model/tool_shed_install/migrate',
-            'old_config_file': 'universe_wsgi.ini',
+            'old_config_files': ['universe_wsgi.ini'],
             'config_prefix': 'install_',
             'default_sqlite_file': './database/install.sqlite',
             'config_override': 'GALAXY_INSTALL_CONFIG_',
@@ -39,14 +41,14 @@ DATABASE = {
 }
 
 
-def read_config_file_arg( argv, default, old_default, cwd=None ):
+def read_config_file_arg( argv, default, old_defaults, cwd=None ):
     config_file = None
     if '-c' in argv:
         pos = argv.index( '-c' )
         argv.pop(pos)
         config_file = argv.pop( pos )
-
-    return find_config_file( default, old_default, config_file, cwd=cwd )
+    old_defaults = listify(old_defaults)
+    return find_config_file( default, old_defaults, config_file, cwd=cwd )
 
 
 def get_config( argv, cwd=None ):
@@ -83,14 +85,18 @@ def get_config( argv, cwd=None ):
     database_defaults = DATABASE[ database ]
 
     default = database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE )
-    old_default = database_defaults.get( 'old_config_file' )
-    config_file = read_config_file_arg( argv, default, old_default, cwd=cwd )
+    old_defaults = database_defaults.get( 'old_config_files' )
+    config_file = read_config_file_arg( argv, default, old_defaults, cwd=cwd )
     repo = database_defaults[ 'repo' ]
     config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX )
     config_override = database_defaults.get( 'config_override', 'GALAXY_CONFIG_' )
     default_sqlite_file = database_defaults[ 'default_sqlite_file' ]
-
-    properties = load_app_properties( ini_file=config_file, config_prefix=config_override )
+    if config_file.endswith(".yml") or config_file.endswith(".yml.sample"):
+        config_section = database_defaults.get( 'config_section', None )
+    else:
+        # An .ini file - just let load_app_properties find app:main.
+        config_section = None
+    properties = load_app_properties( config_file=config_file, config_prefix=config_override, config_section=config_section )
 
     if ("%sdatabase_connection" % config_prefix) in properties:
         db_url = properties[ "%sdatabase_connection" % config_prefix ]

lib/galaxy/util/properties.py

@@ -6,14 +6,20 @@ import os
 import os.path
 import sys
 
+import yaml
+
 from six import iteritems
 from six.moves.configparser import ConfigParser
 
+from galaxy.util import listify
 
-def find_config_file(default, old_default, explicit, cwd=None):
+
+def find_config_file(default, old_defaults, explicit, cwd=None):
+    old_defaults = listify(old_defaults)
     if cwd is not None:
         default = os.path.join(cwd, default)
-        old_default = os.path.join(cwd, old_default)
+        for i in range(len(old_defaults)):
+            old_defaults[i] = os.path.join(cwd, old_defaults[i])
         if explicit is not None:
             explicit = os.path.join(cwd, explicit)
 
@@ -23,33 +29,46 @@ def find_config_file(default, old_default, explicit, cwd=None):
         else:
             raise Exception("Problem determining Galaxy's configuration - the specified configuration file cannot be found.")
     else:
-        if not os.path.exists( default ) and os.path.exists( old_default ):
-            config_file = old_default
-        elif os.path.exists( default ):
+        config_file = None
+        if os.path.exists(default):
             config_file = default
-        else:
+        if config_file is None:
+            for old_default in old_defaults:
+                if os.path.exists(old_default):
+                    config_file = old_default
+
+        if config_file is None:
             config_file = default + ".sample"
+
     return config_file
 
 
 def load_app_properties(
     kwds={},
     ini_file=None,
-    ini_section="app:main",
+    ini_section=None,
+    config_file=None,
+    config_section=None,
     config_prefix="GALAXY_CONFIG_"
 ):
     properties = kwds.copy() if kwds else {}
-    if ini_file:
-        defaults = {
-            'here': os.path.dirname(os.path.abspath(ini_file)),
-            '__file__': os.path.abspath(ini_file)
-        }
-        parser = NicerConfigParser(ini_file, defaults=defaults)
-        parser.optionxform = str  # Don't lower-case keys
-        with open(ini_file) as f:
-            parser.read_file(f)
+    if config_file is None:
+        config_file = ini_file
+        config_section = ini_section
 
-        properties.update( dict( parser.items( ini_section ) ) )
+    if config_file:
+        if not config_file.endswith(".yml") and not config_file.endswith(".yml.sample"):
+            if config_section is None:
+                config_section = "app:main"
+            parser = nice_config_parser(config_file)
+            properties.update( dict( parser.items( config_section ) ) )
+        else:
+            if config_section is None:
+                config_section = "galaxy"
+
+            with open(config_file, "r") as f:
+                raw_properties = yaml.load(f)
+            properties = raw_properties[config_section] or {}
 
     override_prefix = "%sOVERRIDE_" % config_prefix
     for key in os.environ:
@@ -64,6 +83,18 @@ def load_app_properties(
     return properties
 
 
+def nice_config_parser(path):
+    defaults = {
+        'here': os.path.dirname(os.path.abspath(path)),
+        '__file__': os.path.abspath(path)
+    }
+    parser = NicerConfigParser(path, defaults=defaults)
+    parser.optionxform = str  # Don't lower-case keys
+    with open(path) as f:
+        parser.read_file(f)
+    return parser
+
+
 class NicerConfigParser(ConfigParser):
 
     def __init__(self, filename, *args, **kw):

lib/galaxy/web/framework/webapp.py

@@ -954,6 +954,24 @@ class GalaxyWebTransaction( base.DefaultWebTransaction,
         return str(template)
 
 
+def build_native_uwsgi_app( paste_factory, config_section ):
+    """uwsgi can load paste factories with --ini-paste, but this builds non-paste uwsgi apps.
+
+    In particular these are useful with --yaml or --json for config."""
+    import uwsgi
+    uwsgi_opt = uwsgi.opt
+    config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
+    if not config_file:
+        # Probably loaded via --ini-paste - expect paste app.
+        return None
+
+    uwsgi_app = paste_factory(uwsgi.opt, load_app_kwds={
+        "config_file": config_file,
+        "config_section": config_section,
+    })
+    return uwsgi_app
+
+
 def build_url_map( app, global_conf, local_conf ):
     from paste.urlmap import URLMap
     from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static

lib/galaxy/webapps/config_manage.py

@@ -0,0 +1,710 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+from collections import namedtuple
+from collections import OrderedDict
+import copy
+import os
+import shutil
+import string
+import sys
+import tempfile
+from textwrap import TextWrapper
+import urllib2
+
+import six
+from six import StringIO
+
+import yaml
+
+try:
+    from pykwalify.core import Core
+except ImportError:
+    Core = None
+
+if __name__ == '__main__':
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)))
+
+from galaxy.util.properties import nice_config_parser
+from galaxy.util import safe_makedirs
+
+DESCRIPTION = "Convert configuration files."
+
+APP_DESCRIPTION = """Application to target for operation (i.e. galaxy, tool_shed, or reports))"""
+DRY_RUN_DESCRIPTION = """If this action modifies files, just print what would be the result and continue."""
+UNKNOWN_OPTION_MESSAGE = "Option [%s] not found in schema - either it is invalid or the Galaxy team hasn't documented it. If invalid, you should manually remove it. If the option is valid but undocumented, please file an issue with the Galaxy team."
+USING_SAMPLE_MESSAGE = "Path [%s] not a file, using sample."
+EXTRA_SERVER_MESSAGE = "Additional server section after [%s] encountered [%s], will be ignored."
+MISSING_FILTER_TYPE_MESSAGE = "Missing filter type for section [%s], it will be ignored."
+UNHANDLED_FILTER_TYPE_MESSAGE = "Unhandled filter type encountered [%s] for section [%s]."
+NO_APP_MAIN_MESSAGE = "No app:main section found, using application defaults throughout."
+YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ")
+RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent="    ", subsequent_indent="    ")
+UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml"
+
+App = namedtuple(
+    "App",
+    ["config_paths", "default_port", "expected_app_factories", "destination", "schema_path", "uwsgi_module"]
+)
+
+UWSGI_OPTIONS = OrderedDict([
+    ('http', {
+        'desc': """The address and port on which to listen.  By default, only listen to localhost ($app_name will not be accessible over the network).  Use '0.0.0.0' to listen on all available network interfaces.""",
+        'default': '127.0.0.1:$default_port',
+        'type': 'str',
+    }),
+    ('threads', {
+        'default': 8,
+        'type': 'int',
+    }),
+    # ('route-uri', {
+    #     'default': '^/proxy/ goto:proxy'
+    # }),
+    # ('route', {
+    #     'default': '.* last:'
+    # }),
+    # ('route-label', {
+    #     'default': 'proxy'
+    # }),
+    # ('route-run', {
+    #     'default': 'rpcvar:TARGET_HOST galaxy_dynamic_proxy_mapper ${HTTP_HOST} ${cookie[galaxysession]}'
+    # }),
+    # ('route-run', {
+    #     'default': "['log:Proxy ${HTTP_HOST} to ${TARGET_HOST}', 'httpdumb:${TARGET_HOST}']",
+    # }),
+    ('http-raw-body', {
+        'default': 'True'
+    }),
+    ('offload-threads', {
+        'default': '8',
+    }),
+    ('module', {
+        'default': '$uwsgi_module',
+        'type': 'str',
+    })
+])
+
+DROP_OPTION_VALUE = object()
+
+
+class _OptionAction(object):
+
+    def converted(self, args, app_desc, key, value):
+        pass
+
+    def lint(self, args, app_desc, key, value):
+        pass
+
+
+class _DeprecatedAndDroppedAction(_OptionAction):
+
+    def converted(self, args, app_desc, key, value):
+        print("Option [%s] has been deprecated and dropped. It is not included in converted configuration." % key)
+        return DROP_OPTION_VALUE
+
+    def lint(self, args, app_desc, key, value):
+        print("Option [%s] has been deprecated. Option should be dropped without replacement." % key)
+
+
+class _PasteAppFactoryAction(_OptionAction):
+
+    def converted(self, args, app_desc, key, value):
+        if value not in app_desc.expected_app_factories:
+            raise Exception("Ending convert process - unknown paste factory encountered [%s]" % value)
+        return DROP_OPTION_VALUE
+
+    def lint(self, args, app_desc, key, value):
+        if value not in app_desc.expected_app_factories:
+            print("Problem - unknown paste app factory encountered [%s]" % value)
+
+
+class _ProductionNotReady(_OptionAction):
+
+    def __init__(self, unsafe_value):
+        self.unsafe_value = unsafe_value
+
+    def lint(self, args, app_desc, key, value):
+        if str(value).lower() == str(self.unsafe_value).lower():
+            template = "Problem - option [%s] should not be set to [%s] in production environments - it is unsafe."
+            message = template % (key, value)
+            print(message)
+
+
+class _HandleFilterWithAction(_OptionAction):
+
+    def converted(self, args, app_desc, key, value):
+        print("filter-with converted to prefixed module load of uwsgi module, dropping from converted configuration")
+        return DROP_OPTION_VALUE
+
+
+OPTION_ACTIONS = {
+    'use_beaker_session': _DeprecatedAndDroppedAction(),
+    'session_type': _DeprecatedAndDroppedAction(),
+    'session_data_dir': _DeprecatedAndDroppedAction(),
+    'session_key': _DeprecatedAndDroppedAction(),
+    'session_secret': _DeprecatedAndDroppedAction(),
+    'paste.app_factory': _PasteAppFactoryAction(),
+    'filter-with': _HandleFilterWithAction(),
+    'debug': _ProductionNotReady(True),
+    'serve_xss_vulnerable_mimetypes': _ProductionNotReady(True),
+    'use_printdebug': _ProductionNotReady(True),
+    'use_interactive': _ProductionNotReady(True),
+    'id_secret': _ProductionNotReady('USING THE DEFAULT IS NOT SECURE!'),
+    'master_api_key': _ProductionNotReady('changethis'),
+}
+
+
+def _app_name(self):
+    return os.path.splitext(os.path.basename(self.destination))[0]
+
+
+def _sample_destination(self):
+    return self.destination + ".sample"
+
+
+def _schema(self):
+    return AppSchema(self)
+
+
+App.app_name = property(_app_name)
+App.sample_destination = property(_sample_destination)
+App.schema = property(_schema)
+
+OptionValue = namedtuple("OptionValue", ["name", "value", "option"] )
+
+
+UNKNOWN_OPTION = {
+    "type": "str",
+    "required": False,
+    "unknown_option": True,
+    "desc": "Unknown option, may want to remove or report to Galaxy team."
+}
+
+OPTION_DEFAULTS = {
+    "type": "str",
+    "unknown_option": False,
+    "default": None,
+    "desc": None,
+}
+
+
+class Schema(object):
+
+    def __init__(self, mapping):
+        self.app_schema = mapping
+
+    def get_app_option(self, name):
+        try:
+            raw_option = self.app_schema[name]
+        except KeyError:
+            raw_option = UNKNOWN_OPTION
+        option = OPTION_DEFAULTS.copy()
+        option.update(raw_option)
+        return option
+
+
+class AppSchema(Schema):
+
+    def __init__(self, app_desc):
+        schema_path = app_desc.schema_path
+        app_name = app_desc.app_name
+        with open(schema_path, "r") as f:
+            config_all = _ordered_load(f)
+        self.raw_schema = config_all
+        app_schema = config_all["mapping"][app_name]
+        super(AppSchema, self).__init__(app_schema["mapping"])
+
+    def get_app_option(self, name):
+        try:
+            raw_option = self.app_schema[name]
+        except KeyError:
+            raw_option = UNKNOWN_OPTION
+        option = OPTION_DEFAULTS.copy()
+        option.update(raw_option)
+        return option
+
+
+GALAXY_APP = App(
+    ["universe_wsgi.ini", "config/galaxy.ini"],
+    "8080",
+    ["galaxy.web.buildapp:app_factory"],  # TODO: Galaxy could call factory a few different things and they'd all be fine.
+    "config/galaxy.yml",
+    "lib/galaxy/webapps/galaxy/config_schema.yml",
+    'galaxy.webapps.galaxy.buildapp:uwsgi_app()',
+)
+SHED_APP = App(
+    ["tool_shed_wsgi.ini", "config/tool_shed.ini"],
+    "9009",
+    ["galaxy.webapps.tool_shed.buildapp:app_factory"],
+    "config/tool_shed.yml",
+    "lib/galaxy/webapps/tool_shed/config_schema.yml",
+    'galaxy.webapps.tool_shed.buildapp:uwsgi_app()',
+)
+REPORTS_APP = App(
+    ["reports_wsgi.ini", "config/reports.ini"],
+    "9001",
+    ["galaxy.webapps.reports.buildapp:app_factory"],
+    "config/reports.yml",
+    "lib/galaxy/webapps/reports/config_schema.yml",
+    'galaxy.webapps.reports.buildapp:uwsgi_app()',
+)
+APPS = {"galaxy": GALAXY_APP, "tool_shed": SHED_APP, "reports": REPORTS_APP}
+
+
+def main(argv=None):
+    """Entry point for conversion process."""
+    if argv is None:
+        argv = sys.argv[1:]
+    args = _arg_parser().parse_args(argv)
+    app_name = args.app
+    app_desc = APPS.get(app_name, None)
+    action = args.action
+    action_func = ACTIONS[action]
+    action_func(args, app_desc)
+
+
+def _arg_parser():
+    parser = argparse.ArgumentParser(description=DESCRIPTION)
+    parser.add_argument('action', metavar='ACTION', type=str,
+                        choices=ACTIONS.keys(),
+                        help='action to perform')
+    parser.add_argument('app', metavar='APP', type=str, nargs="?",
+                        help=APP_DESCRIPTION)
+    parser.add_argument('--add-comments', default=False, action="store_true")
+    parser.add_argument('--dry-run', default=False, action="store_true",
+                        help=DRY_RUN_DESCRIPTION)
+    parser.add_argument('--galaxy_root', default=".", type=str)
+    return parser
+
+
+def _to_rst(args, app_desc, heading_level="~"):
+    rst = StringIO()
+    schema = app_desc.schema
+    for key, value in schema.app_schema.items():
+        default = None if "default" not in value else value["default"]
+        option = schema.get_app_option(key)
+        option_value = OptionValue(key, default, option)
+        _write_option_rst(args, rst, key, heading_level, option_value)
+    print(rst.getvalue())
+
+
+def _write_option_rst(args, rst, key, heading_level, option_value):
+    title = "``%s``" % key
+    heading = heading_level * len(title)
+    rst.write("%s\n%s\n%s\n\n" % (heading, title, heading))
+    option, value = _parse_option_value(option_value)
+    desc = option["desc"]
+    rst.write(":Description:\n")
+    rst.write("\n".join(RST_DESCRIPTION_WRAPPER.wrap(desc)))
+    rst.write("\n")
+    type = option.get("type", None)
+    default = option.get("default", "*null*")
+    rst.write(":Default: %s\n" % default)
+    if type:
+        rst.write(":Type: %s\n" % type)
+    rst.write("\n\n")
+
+
+def _build_uwsgi_schema(args, app_desc):
+    req = urllib2.Request('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst')
+    response = urllib2.urlopen(req)
+    rst_options = response.read()
+    last_line = None
+    current_opt = None
+
+    options = OrderedDict({})
+    option = None
+    for line in rst_options.splitlines():
+        line = line.strip()
+        dots = "*" * len(line)
+        if line and (line == dots):
+            current_opt = last_line
+            option = {
+                'type': 'any',
+            }
+            options[current_opt] = option
+
+        if line.startswith("``parser``"):
+            parser = line.split(":", 1)[1].strip()
+            if parser == "uwsgi_opt_set_int":
+                option["type"] = "int"
+            # TODO: disptch on parser...
+        elif line.startswith("``help``"):
+            option["desc"] = line.split(":", 1)[1]
+
+        last_line = line
+    schema = {
+        "type": "map",
+        "desc": "uwsgi definition, see http://uwsgi-docs.readthedocs.io/en/latest/Options.html",
+        "mapping": options
+    }
+    path = os.path.join(args.galaxy_root, UWSGI_SCHEMA_PATH)
+    contents = _ordered_dump(schema)
+    _write_to_file(args, contents, path)
+
+
+def _find_config(args, app_desc):
+    path = os.path.join(args.galaxy_root, app_desc.destination)
+    if not os.path.exists(path):
+        path = None
+
+        for possible_ini_config_rel in app_desc.config_paths:
+            possible_ini_config = os.path.join(args.galaxy_root, possible_ini_config_rel)
+            if os.path.exists(possible_ini_config):
+                path = possible_ini_config
+
+    if path is None:
+        _warn(USING_SAMPLE_MESSAGE % path)
+        path = os.path.join(args.galaxy_root, app_desc.sample_destination)
+
+    return path
+
+
+def _find_app_options(app_desc, path):
+    """Load app (as opposed to server) options from specified path.
+
+    Supplied ``path`` may be either YAML or ini file.
+    """
+    if _is_ini(path):
+        p = nice_config_parser(path)
+        app_items = _find_app_options_from_config_parser(p)
+    else:
+        raw_config = _order_load_path(path)
+        app_items = raw_config.get(app_desc.app_name, None) or {}
+    return app_items
+
+
+def _find_app_options_from_config_parser(p):
+    if not p.has_section("app:main"):
+        _warn(NO_APP_MAIN_MESSAGE)
+        app_items = OrderedDict()
+    else:
+        app_items = OrderedDict(p.items("app:main"))
+
+    return app_items
+
+
+def _lint(args, app_desc):
+    path = _find_config(args, app_desc)
+    if not os.path.exists(path):
+        raise Exception("Expected configuration file [%s] not found." % path)
+    app_items = _find_app_options(app_desc, path)
+    for key, value in app_items.items():
+        option_action = OPTION_ACTIONS.get(key)
+        if option_action is not None:
+            option_action.lint(args, app_desc, key, value)
+
+
+def _validate(args, app_desc):
+    path = _find_config(args, app_desc)
+    with open(path, "r") as f:
+        # Allow empty mapping (not allowed by pykawlify)
+        raw_config = _order_load_path(f)
+        if raw_config.get(app_desc.app_name, None) is None:
+            raw_config[app_desc.app_name] = {}
+            config_p = tempfile.NamedTemporaryFile(delete=False, suffix=".yml")
+            _ordered_dump(raw_config, config_p)
+            config_p.flush()
+            path = config_p.name
+
+    fp = tempfile.NamedTemporaryFile(delete=False, suffix=".yml")
+    _ordered_dump(app_desc.schema.raw_schema, fp)
+    fp.flush()
+    name = fp.name
+    if Core is None:
+        raise Exception("Cannot validate file, pykwalify is not installed.")
+    c = Core(
+        source_file=path,
+        schema_files=[name],
+    )
+    c.validate()
+
+
+class PrefixFilter(object):
+
+    def __init__(self, name, prefix):
+        self.name = name
+        self.prefix = prefix
+
+
+class GzipFilter(object):
+
+    def __init__(self, name):
+        self.name = name
+
+
+def _run_conversion(args, app_desc):
+    ini_config = _find_config(args, app_desc)
+    if ini_config and not _is_ini(ini_config):
+        _warn("Cannot convert YAML file %s, this option is only for ini config files." % ini_config)
+        sys.exit(1)
+    elif not ini_config:
+        _warn("Failed to find a config to convert - exiting without changes.")
+        sys.exit(1)
+
+    p = nice_config_parser(ini_config)
+    server_section = None
+    filters = {}
+    for section in p.sections():
+        if section.startswith("server:"):
+            if server_section:
+                _warn(EXTRA_SERVER_MESSAGE % (server_section, section))
+            else:
+                server_section = section
+
+        if section.startswith("filter:"):
+            filter_name = section[len("filter:"):]
+            filter_type = p.get(section, "use")
+            if filter_type is None:
+                MISSING_FILTER_TYPE_MESSAGE
+                message = EXTRA_SERVER_MESSAGE % section
+                _warn(message)
+                continue
+
+            if filter_type == "egg:PasteDeploy#prefix":
+                prefix = p.get(section, "prefix")
+                filters[filter_name] = PrefixFilter(filter_name, prefix)
+            elif filter_type == "egg:Paste#gzip":
+                filters[filter_name] = GzipFilter(filter_name)
+            else:
+                message = UNHANDLED_FILTER_TYPE_MESSAGE % (filter_type, section)
+                _warn(message)
+                continue
+
+    if not server_section:
+        _warn("No server section found, using default uwsgi server definition.")
+        server_config = OrderedDict()
+    else:
+        server_config = OrderedDict(p.items(server_section))
+
+    app_items = _find_app_options_from_config_parser(p)
+    applied_filters = []
+    if filters:
+        for key, value in app_items.items():
+            if key == "filter-with":
+                if value in filters:
+                    applied_filters.append(filters[value])
+                else:
+                    _warn("Unknown filter found [%s], exiting..." % value)
+                    sys.exit(1)
+
+    uwsgi_dict = _server_paste_to_uwsgi(app_desc, server_config, applied_filters)
+
+    app_dict = OrderedDict({})
+    schema = app_desc.schema
+    for key, value in app_items.items():
+        if key in ["__file__", "here"]:
+            continue
+
+        if key in OPTION_ACTIONS:
+            option_action = OPTION_ACTIONS.get(key)
+            value = option_action.converted(args, app_desc, key, value)
+
+        if value is DROP_OPTION_VALUE:
+            continue
+
+        option = schema.get_app_option(key)
+        if option["unknown_option"]:
+            _warn(UNKNOWN_OPTION_MESSAGE % key)
+
+        option_value = OptionValue(key, value, option)
+        app_dict[key] = option_value
+
+    f = StringIO()
+    _write_section(args, f, "uwsgi", uwsgi_dict)
+    _write_section(args, f, app_desc.app_name, app_dict)
+    destination = os.path.join(args.galaxy_root, app_desc.destination)
+    _replace_file(args, f, app_desc, ini_config, destination)
+
+
+def _is_ini(path):
+    return path.endswith(".ini") or path.endswith(".ini.sample")
+
+
+def _replace_file(args, f, app_desc, from_path, to_path):
+    _write_to_file(args, f, to_path)
+    backup_path = "%s.backup" % from_path
+    print("Moving [%s] to [%s]" % (from_path, backup_path))
+    if args.dry_run:
+        print("... skipping because --dry-run is enabled.")
+    else:
+        shutil.move(from_path, backup_path)
+
+
+def _build_sample_yaml(args, app_desc):
+    schema = app_desc.schema
+    f = StringIO()
+    options = copy.deepcopy(UWSGI_OPTIONS)
+    for key, value in options.items():
+        for field in ["desc", "default"]:
+            if field not in value:
+                continue
+            field_value = value[field]
+            if not isinstance(field_value, six.string_types):
+                continue
+
+            new_field_value = string.Template(field_value).safe_substitute(**{
+                'default_port': str(app_desc.default_port),
+                'app_name': app_desc.app_name,
+                'uwsgi_module': app_desc.uwsgi_module,
+            })
+            value[field] = new_field_value
+    _write_sample_section(args, f, 'uwsgi', Schema(options), as_comment=False)
+    _write_sample_section(args, f, app_desc.app_name, schema)
+    destination = os.path.join(args.galaxy_root, app_desc.sample_destination)
+    _write_to_file(args, f, destination)
+
+
+def _write_to_file(args, f, path):
+    dry_run = args.dry_run
+    if hasattr(f, "getvalue"):
+        contents = f.getvalue()
+    else:
+        contents = f
+    contents_indented = "\n".join([" |%s" % l for l in contents.splitlines()])
+    print("Writing the file contents:\n%s\nto %s" % (contents_indented, path))
+    if dry_run:
+        print("... skipping because --dry-run is enabled.")
+    else:
+        safe_makedirs(os.path.dirname(path))
+        with open(path, "w") as to_f:
+            to_f.write(contents)
+
+
+def _order_load_path(path):
+    """Load (with ``_ordered_load``) on specified path (a YAML file)."""
+    with open(path, "r") as f:
+        # Allow empty mapping (not allowed by pykawlify)
+        raw_config = _ordered_load(f)
+        return raw_config
+
+
+def _write_sample_section(args, f, section_header, schema, as_comment=True):
+    _write_header(f, section_header)
+    for key, value in schema.app_schema.items():
+        default = None if "default" not in value else value["default"]
+        option = schema.get_app_option(key)
+        option_value = OptionValue(key, default, option)
+        _write_option(args, f, key, option_value, as_comment=as_comment)
+
+
+def _write_section(args, f, section_header, section_dict):
+    _write_header(f, section_header)
+    for key, option_value in section_dict.items():
+        _write_option(args, f, key, option_value)
+
+
+def _write_header(f, section_header):
+    f.write("%s:\n\n" % section_header)
+
+
+def _write_option(args, f, key, option_value, as_comment=False):
+    option, value = _parse_option_value(option_value)
+    desc = option["desc"]
+    comment = ""
+    if desc and args.add_comments:
+        comment = "\n".join(YAML_COMMENT_WRAPPER.wrap(desc))
+        comment += "\n"
+    as_comment_str = "#" if as_comment else ""
+    key_val_str = yaml.dump({key: value}).lstrip("{").rstrip("\n}")
+    lines = "%s%s%s" % (comment, as_comment_str, key_val_str)
+    lines_idented = "\n".join([("  %s" % l) for l in lines.split("\n")])
+    f.write("%s\n\n" % lines_idented)
+
+
+def _parse_option_value(option_value):
+    if isinstance(option_value, OptionValue):
+        option = option_value.option
+        value = option_value.value
+    else:
+        value = option_value
+        option = OPTION_DEFAULTS
+    return option, value
+
+
+def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
+    uwsgi_dict = OrderedDict()
+    port = server_config.get("port", app_desc.default_port)
+    host = server_config.get("host", "127.0.0.1")
+
+    if server_config.get("use", "egg:Paste#http") != "egg:Paste#http":
+        raise Exception("Unhandled paste server 'use' value [%s], file must be manually migrate.")
+
+    uwsgi_dict["http"] = "%s:%s" % (host, port)
+    # default changing from 10 to 8
+    uwsgi_dict["threads"] = int(server_config.get("threadpool_workers", 8))
+    # required for static...
+    uwsgi_dict["http-raw-body"] = True
+    uwsgi_dict["offload-threads"] = 8
+
+    # Handle paste filters during conversion.
+    prefix = None
+    for applied_filter in applied_filters:
+        if isinstance(applied_filter, PrefixFilter):
+            prefix = applied_filter.prefix
+            break
+        elif isinstance(applied_filter, GzipFilter):
+            uwsgi_dict["http-auto-gzip"] = True
+
+    if prefix:
+        uwsgi_dict["mount"] = "%s=%s" % (prefix, app_desc.uwsgi_module)
+        uwsgi_dict["manage-script-name"] = True
+    else:
+        uwsgi_dict["module"] = app_desc.uwsgi_module
+    return uwsgi_dict
+
+
+def _warn(message):
+    print("WARNING: %s" % message)
+
+
+def _ordered_load(stream):
+
+    class OrderedLoader(yaml.Loader):
+
+        def __init__(self, stream):
+            self._root = os.path.split(stream.name)[0]
+            super(OrderedLoader, self).__init__(stream)
+
+        def include(self, node):
+            filename = os.path.join(self._root, self.construct_scalar(node))
+            with open(filename, 'r') as f:
+                return yaml.load(f, OrderedLoader)
+
+    def construct_mapping(loader, node):
+        loader.flatten_mapping(node)
+        return OrderedDict(loader.construct_pairs(node))
+
+    OrderedLoader.add_constructor(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        construct_mapping)
+    OrderedLoader.add_constructor('!include', OrderedLoader.include)
+
+    return yaml.load(stream, OrderedLoader)
+
+
+def _ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds):
+    class OrderedDumper(Dumper):
+        pass
+
+    def _dict_representer(dumper, data):
+        return dumper.represent_mapping(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+            data.items())
+    OrderedDumper.add_representer(OrderedDict, _dict_representer)
+    return yaml.dump(data, stream, OrderedDumper, **kwds)
+
+
+ACTIONS = {
+    "convert": _run_conversion,
+    "build_sample_yaml": _build_sample_yaml,
+    "validate": _validate,
+    "lint": _lint,
+    "build_uwsgi_yaml": _build_uwsgi_schema,
+    "build_rst": _to_rst,
+}
+
+
+if __name__ == '__main__':
+    main()

lib/galaxy/webapps/galaxy/buildapp.py

@@ -159,6 +159,8 @@ def paste_app_factory( global_conf, **kwargs ):
 
 
 def uwsgi_app_factory():
+    # TODO: synchronize with galaxy.web.framework.webapp.build_native_uwsgi_app - should
+    # at least be using nice_config_parser for instance.
     import uwsgi
     root = os.path.abspath(uwsgi.opt.get('galaxy_root', os.getcwd()))
     config_file = uwsgi.opt.get('galaxy_config_file', os.path.join(root, 'config', 'galaxy.ini'))

lib/galaxy/webapps/reports/buildapp.py

@@ -51,11 +51,12 @@ def add_ui_controllers( webapp, app ):
                     webapp.add_ui_controller( name, T( app ) )
 
 
-def app_factory( global_conf, **kwargs ):
+def app_factory( global_conf, load_app_kwds={}, **kwargs ):
     """Return a wsgi application serving the root object"""
     # Create the Galaxy application unless passed in
     kwargs = load_app_properties(
-        kwds=kwargs
+        kwds=kwargs,
+        **load_app_kwds
     )
     if 'app' in kwargs:
         app = kwargs.pop( 'app' )
@@ -149,3 +150,7 @@ def wrap_in_middleware( app, global_conf, application_stack, **local_conf ):
 def wrap_in_static( app, global_conf, **local_conf ):
     urlmap, _ = galaxy.web.framework.webapp.build_url_map( app, global_conf, local_conf )
     return urlmap
+
+
+def uwsgi_app():
+    return galaxy.web.framework.webapp.build_native_uwsgi_app( app_factory, "reports" )

lib/galaxy/webapps/reports/config_schema.yml

@@ -0,0 +1,103 @@
+type: map
+mapping:
+  uwsgi: !include uwsgi_schema.yml
+  reports:
+    type: map
+    required: true
+    desc: |
+      Galaxy Reports configuration options.
+    mapping:
+      filter-with:
+        type: str
+        default: proxy-prefix
+        desc: |
+          If running behind a proxy server and Galaxy is served from a subdirectory,
+          enable the proxy-prefix filter and set the prefix in the
+          [filter:proxy-prefix] section above.
+      
+      cookie_path:
+        type: str
+        desc: |
+          If proxy-prefix is enabled and you're running more than one Galaxy instance
+          behind one hostname, you will want to set this to the same path as the prefix
+          in the filter above.  This value becomes the "path" attribute set in the
+          cookie so the cookies from each instance will not clobber each other.
+      
+      log_level:
+        type: str
+        default: DEBUG
+        desc: |
+          Verbosity of console log messages.  Acceptable values can be found here:
+          https://docs.python.org/2/library/logging.html#logging-levels
+      
+      database_connection:
+        type: str
+        default: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
+        desc: |
+          Database connection
+          Galaxy reports are intended for production Galaxy instances, so sqlite (and the default value
+          below) is not supported. An SQLAlchemy connection string should be used specify an external
+          database.
+      
+      file_path:
+        type: str
+        default: database/files
+        desc: |
+          Where dataset files are saved
+          Temporary storage for additional datasets, this should be shared through the cluster
+      
+      new_file_path:
+        type: str
+        default: database/tmp
+        desc: |
+          Where dataset files are saved
+          Temporary storage for additional datasets, this should be shared through the cluster
+      
+      template_cache_path:
+        type: str
+        default: database/compiled_templates/reports
+        desc: |
+          Mako templates are compiled as needed and cached for reuse, this directory is
+          used for the cache
+      
+      debug:
+        type: bool
+        default: false
+        desc: |
+          Configuration for debugging middleware
+      
+      use_lint:
+        type: bool
+        default: false
+        desc: |
+          Check for WSGI compliance.
+      
+      use_interactive:
+        type: bool
+        default: true
+        desc: |
+          NEVER enable this on a public site (even test or QA)
+      
+      use_heartbeat:
+        type: bool
+        default: true
+        desc: |
+          Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!)
+      
+      use_profile:
+        type: bool
+        default: true
+        desc: |
+          Profiling middleware (cProfile based)
+      
+      smtp_server:
+        type: str
+        default: yourserver@yourfacility.edu
+        desc: |
+          Mail
+      
+      error_email_to:
+        type: str
+        default: your_bugs@bx.psu.edu
+        desc: |
+          Mail

lib/galaxy/webapps/reports/uwsgi_schema.yml

@@ -0,0 +1 @@
+../uwsgi_schema.yml

lib/galaxy/webapps/tool_shed/buildapp.py

@@ -53,12 +53,13 @@ def add_ui_controllers( webapp, app ):
                     webapp.add_ui_controller( name, T( app ) )
 
 
-def app_factory( global_conf, **kwargs ):
+def app_factory( global_conf, load_app_kwds={}, **kwargs ):
     """Return a wsgi application serving the root object"""
     # Create the Galaxy tool shed application unless passed in
     kwargs = load_app_properties(
         kwds=kwargs,
-        config_prefix='TOOL_SHED_CONFIG_'
+        config_prefix='TOOL_SHED_CONFIG_',
+        **load_app_kwds
     )
     if 'app' in kwargs:
         app = kwargs.pop( 'app' )
@@ -309,3 +310,7 @@ def _map_redirects( mapper ):
 
     mapper.redirect( "/repository/status_for_installed_repository", "/api/repositories/updates/", _redirect_code="301 Moved Permanently", conditions=dict( function=forward_qs ) )
     return mapper
+
+
+def uwsgi_app():
+    return galaxy.web.framework.webapp.build_native_uwsgi_app( app_factory, "tool_shed" )

lib/galaxy/webapps/tool_shed/config_schema.yml

@@ -0,0 +1,387 @@
+type: map
+mapping:
+  uwsgi: !include uwsgi_schema.yml
+  tool_shed:
+    type: map
+    required: true
+    desc: |
+      Tool Shed configuration options.
+    mapping:
+      log_level:
+        type: str
+        default: DEBUG
+        required: false
+        desc: |
+          Verbosity of console log messages.  Acceptable values can be found here:
+          https://docs.python.org/2/library/logging.html#logging-levels
+      
+      database_connection:
+        type: str
+        default: sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE
+        required: false
+        desc: |
+          By default, the Tool Shed uses a SQLite database at 'database/community.sqlite'.  You
+          may use a SQLAlchemy connection string to specify an external database
+          instead.  This string takes many options which are explained in detail in the
+          config file documentation.
+      
+      hgweb_config_dir:
+        type: str
+        required: false
+        desc: |
+          Where the hgweb.config file is stored.
+          The default is the Galaxy installation directory.
+      
+      file_path:
+        type: str
+        default: database/community_files
+        required: false
+        desc: |
+          Where tool shed repositories are stored.
+      
+      new_file_path:
+        type: str
+        default: database/tmp
+        required: false
+        desc: |
+          Temporary storage for additional datasets,
+          this should be shared through the cluster
+      
+      builds_file_path:
+        type: str
+        default: tool-data/shared/ucsc/builds.txt
+        required: false
+        desc: |
+          File containing old-style genome builds
+      
+      pretty_datetime_format:
+        type: str
+        default: $locale (UTC)
+        required: false
+        desc: |
+          Format string used when showing date and time information.
+          The string may contain:
+          - the directives used by Python time.strftime() function (see
+            https://docs.python.org/2/library/time.html#time.strftime ),
+          - $locale (complete format string for the server locale),
+          - $iso8601 (complete format string as specified by ISO 8601 international
+            standard).
+      
+      toolshed_search_on:
+        type: bool
+        default: true
+        required: false
+        desc: |
+          -- Repository and Tool search
+          Using the script located at scripts/build_ts_whoosh_index.py
+          you can generate search index and allow full text API searching over
+          the repositories and tools within the Tool Shed given that you specify
+          the following two config options.
+      
+      whoosh_index_dir:
+        type: str
+        default: database/toolshed_whoosh_indexes
+        required: false
+        desc: |
+          -- Repository and Tool search
+          Using the script located at scripts/build_ts_whoosh_index.py
+          you can generate search index and allow full text API searching over
+          the repositories and tools within the Tool Shed given that you specify
+          the following two config options.
+      
+      repo_name_boost:
+        type: float
+        default: 0.9
+        required: false
+        desc: |
+          For searching repositories at /api/repositories:
+      
+      repo_description_boost:
+        type: float
+        default: 0.6
+        required: false
+        desc: |
+          For searching repositories at /api/repositories:
+      
+      repo_long_description_boost:
+        type: float
+        default: 0.5
+        required: false
+        desc: |
+          For searching repositories at /api/repositories:
+      
+      repo_homepage_url_boost:
+        type: float
+        default: 0.3
+        required: false
+        desc: |
+          For searching repositories at /api/repositories:
+      
+      repo_remote_repository_url_boost:
+        type: float
+        default: 0.2
+        required: false
+        desc: |
+          For searching repositories at /api/repositories:
+      
+      repo_owner_username_boost:
+        type: float
+        default: 0.3
+        required: false
+        desc: |
+          For searching repositories at /api/repositories:
+      
+      tool_name_boost:
+        type: float
+        default: 1.2
+        required: false
+        desc: |
+          For searching tools at /api/tools
+      
+      tool_description_boost:
+        type: float
+        default: 0.6
+        required: false
+        desc: |
+          For searching tools at /api/tools
+      
+      tool_help_boost:
+        type: float
+        default: 0.4
+        required: false
+        desc: |
+          For searching tools at /api/tools
+      
+      tool_repo_owner_username:
+        type: float
+        default: 0.3
+        required: false
+        desc: |
+          For searching tools at /api/tools
+      
+      ga_code:
+        type: str
+        required: false
+        desc: |
+          You can enter tracking code here to track visitor's behavior
+          through your Google Analytics account. Example: UA-XXXXXXXX-Y
+      
+      id_secret:
+        type: str
+        default: changethisinproductiontoo
+        required: false
+        desc: |
+          The Tool Shed encodes various internal values when these values will be output in
+          some format (for example, in a URL or cookie).  You should set a key to be
+          used by the algorithm that encodes and decodes these values.  It can be any
+          string.  If left unchanged, anyone could construct a cookie that would grant
+          them access to others' sessions.
+          One simple way to generate a value for this is with the shell command:
+            python -c 'import time; print time.time()' | md5sum | cut -f 1 -d ' '
+      
+      use_remote_user:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          User authentication can be delegated to an upstream proxy server (usually
+          Apache).  The upstream proxy should set a REMOTE_USER header in the request.
+          Enabling remote user disables regular logins.  For more information, see:
+          https://wiki.galaxyproject.org/Admin/Config/ApacheProxy
+      
+      remote_user_secret:
+        type: str
+        default: changethisinproductiontoo
+        required: false
+        desc: |
+          If use_remote_user is enabled, anyone who can log in to the Galaxy host may
+          impersonate any other user by simply sending the appropriate header. Thus a
+          secret shared between the upstream proxy server, and Galaxy is required.
+          If anyone other than the Galaxy user is using the server, then apache/nginx
+          should pass a value in the header 'GX_SECRET' that is identical the one below
+      
+      debug:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Configuration for debugging middleware
+      
+      use_lint:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Check for WSGI compliance.
+      
+      use_printdebug:
+        type: bool
+        default: true
+        required: false
+        desc: |
+          Intercept print statements and show them on the returned page.
+      
+      use_interactive:
+        type: bool
+        default: true
+        required: false
+        desc: |
+          NEVER enable this on a public site (even test or QA)
+      
+      admin_users:
+        type: str
+        required: false
+        desc: |
+          Administrative users - set this to a comma-separated list of valid Tool Shed
+          users (email addresses).  These users will have access to the Admin section
+          of the server, and will have access to create users, groups, roles,
+          libraries, and more.
+      
+      require_login:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Force everyone to log in (disable anonymous access)
+      
+      smtp_server:
+        type: str
+        default: smtp.your_tool_shed_server
+        required: false
+        desc: |
+          For use by email messages sent from the tool shed
+      
+      email_from:
+        type: str
+        default: your_tool_shed_email@server
+        required: false
+        desc: |
+          For use by email messages sent from the tool shed
+      
+      smtp_username:
+        type: str
+        required: false
+        desc: |
+          If your SMTP server requires a username and password, you can provide them
+          here (password in cleartext here, but if your server supports STARTTLS it
+          will be sent over the network encrypted).
+      
+      smtp_password:
+        type: str
+        required: false
+        desc: |
+          If your SMTP server requires a username and password, you can provide them
+          here (password in cleartext here, but if your server supports STARTTLS it
+          will be sent over the network encrypted).
+      
+      smtp_ssl:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          If your SMTP server requires SSL from the beginning of the connection
+      
+      support_url:
+        type: str
+        default: https://wiki.galaxyproject.org/Support
+        required: false
+        desc: |
+          The URL linked by the "Support" link in the "Help" menu.
+      
+      mailing_join_addr:
+        type: str
+        default: galaxy-announce-join@bx.psu.edu
+        required: false
+        desc: |
+          Address to join mailing list
+      
+      use_heartbeat:
+        type: bool
+        default: true
+        required: false
+        desc: |
+          Write thread status periodically to 'heartbeat.log' (careful, uses disk
+           space rapidly!)
+      
+      use_profile:
+        type: bool
+        default: true
+        required: false
+        desc: |
+          Profiling middleware (cProfile based)
+      
+      enable_galaxy_flavor_docker_image:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Enable creation of Galaxy flavor Docker Image
+      
+      message_box_visible:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Show a message box under the masthead.
+      
+      message_box_content:
+        type: str
+        required: false
+        desc: |
+          Show a message box under the masthead.
+      
+      message_box_class:
+        type: str
+        default: info
+        required: false
+        desc: |
+          Show a message box under the masthead.
+      
+      static_enabled:
+        type: bool
+        default: true
+        required: false
+        desc: |
+          Serving static files (needed if running standalone)
+      
+      static_cache_time:
+        type: int
+        default: 360
+        required: false
+        desc: |
+          Serving static files (needed if running standalone)
+      
+      static_dir:
+        type: str
+        default: static/
+        required: false
+        desc: |
+          Serving static files (needed if running standalone)
+      
+      static_images_dir:
+        type: str
+        default: static/images
+        required: false
+        desc: |
+          Serving static files (needed if running standalone)
+      
+      static_favicon_dir:
+        type: str
+        default: static/favicon.ico
+        required: false
+        desc: |
+          Serving static files (needed if running standalone)
+      
+      static_scripts_dir:
+        type: str
+        default: static/scripts/
+        required: false
+        desc: |
+          Serving static files (needed if running standalone)
+      
+      static_style_dir:
+        type: str
+        default: static/style/blue
+        required: false
+        desc: |
+          Serving static files (needed if running standalone)

lib/galaxy/webapps/tool_shed/uwsgi_schema.yml

@@ -0,0 +1 @@
+../uwsgi_schema.yml

run.sh

@@ -2,6 +2,8 @@
 
 cd "$(dirname "$0")"
 
+. ./scripts/common_startup_functions.sh
+
 # If there is a file that defines a shell environment specific to this
 # instance of Galaxy, source the file.
 if [ -z "$GALAXY_LOCAL_ENV_FILE" ];
@@ -14,72 +16,13 @@ then
     . $GALAXY_LOCAL_ENV_FILE
 fi
 
-INITIALIZE_TOOL_DEPENDENCIES=1  # Install Conda if needed.
-# Pop args meant for common_startup.sh
-while :
-do
-    case "$1" in
-        --skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip|--replace-pip)
-            common_startup_args="$common_startup_args $1"
-            shift
-            ;;
-        --skip-tool-dependency-initialization)
-            INITIALIZE_TOOL_DEPENDENCIES=0
-            shift
-            ;;
-        --skip-venv)
-            skip_venv=1
-            common_startup_args="$common_startup_args $1"
-            shift
-            ;;
-        --stop-daemon)
-            common_startup_args="$common_startup_args $1"
-            paster_args="$paster_args $1"
-            stop_daemon_arg_set=1
-            shift
-            ;;
-        --daemon|--restart|restart)
-            if [ "$1" = "--restart" ]
-            then
-                paster_args="$paster_args restart"
-            else
-                paster_args="$paster_args $1"
-            fi
-
-            daemon_or_restart_arg_set=1
-            shift
-            ;;
-        --wait)
-            wait_arg_set=1
-            shift
-            ;;
-        "")
-            break
-            ;;
-        *)
-            paster_args="$paster_args $1"
-            shift
-            ;;
-    esac
-done
-
 ./scripts/common_startup.sh $common_startup_args || exit 1
 
-# If there is a .venv/ directory, assume it contains a virtualenv that we
-# should run this instance in.
-GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-.venv}"
-if [ -d "$GALAXY_VIRTUAL_ENV" -a -z "$skip_venv" ];
-then
-    [ -n "$PYTHONPATH" ] && { echo 'Unsetting $PYTHONPATH'; unset PYTHONPATH; }
-    echo "Activating virtualenv at $GALAXY_VIRTUAL_ENV"
-    . "$GALAXY_VIRTUAL_ENV/bin/activate"
-fi
+parse_common_args $@
 
-# If you are using --skip-venv we assume you know what you are doing but warn
-# in case you don't.
-[ -n "$PYTHONPATH" ] && echo 'WARNING: $PYTHONPATH is set, this can cause problems importing Galaxy dependencies'
+run_common_start_up
 
-python ./scripts/check_python.py || exit 1
+setup_python
 
 if [ ! -z "$GALAXY_RUN_WITH_TEST_TOOLS" ];
 then

run_reports.sh

@@ -9,17 +9,27 @@
 # argument to this will cause Galaxy's database and path parameters
 # from galaxy.ini to be copied over into reports.ini.
 
-cd `dirname $0`
+cd "$(dirname "$0")"
 
-./scripts/common_startup.sh --skip-samples
+GALAXY_REPORTS_PID=${GALAXY_REPORTS_PID:-reports_webapp.pid}
+GALAXY_REPORTS_LOG=${GALAXY_REPORTS_LOG:-reports_webapp.log}
+PID_FILE=$GALAXY_REPORTS_PID
+LOG_FILE=$GALAXY_REPORTS_LOG
 
-: ${GALAXY_VIRTUAL_ENV:=.venv}
+. ./scripts/common_startup_functions.sh
 
-if [ -d "$GALAXY_VIRTUAL_ENV" ];
+if [ "$1" = "--sync-config" ];
 then
-    . "$GALAXY_VIRTUAL_ENV/bin/activate"
+    python ./scripts/sync_reports_config.py
+    shift
 fi
 
+parse_common_args $@
+
+run_common_start_up
+
+setup_python
+
 if [ -z "$GALAXY_REPORTS_CONFIG" ]; then
     if [ -f reports_wsgi.ini ]; then
         GALAXY_REPORTS_CONFIG=reports_wsgi.ini
@@ -27,24 +37,17 @@ if [ -z "$GALAXY_REPORTS_CONFIG" ]; then
         GALAXY_REPORTS_CONFIG=config/reports_wsgi.ini
     elif [ -f config/reports.ini ]; then
         GALAXY_REPORTS_CONFIG=config/reports.ini
+    elif [ -f config/reports.yml ]; then
+        GALAXY_REPORTS_CONFIG=config/reports.yml
     else
-        GALAXY_REPORTS_CONFIG=config/reports.ini.sample
+        GALAXY_REPORTS_CONFIG=config/reports.yml.sample
     fi
     export GALAXY_REPORTS_CONFIG
 fi
 
-GALAXY_REPORTS_PID=${GALAXY_REPORTS_PID:-reports_webapp.pid}
-GALAXY_REPORTS_LOG=${GALAXY_REPORTS_LOG:-reports_webapp.log}
-
 if [ -n "$GALAXY_REPORTS_CONFIG_DIR" ]; then
     python ./scripts/build_universe_config.py "$GALAXY_REPORTS_CONFIG_DIR" "$GALAXY_REPORTS_CONFIG"
 fi
 
-
-if [ "$1" = "--sync-config" ];
-then
-    python ./scripts/sync_reports_config.py
-    shift
-fi
-
-python ./scripts/paster.py serve "$GALAXY_REPORTS_CONFIG" --pid-file="$GALAXY_REPORTS_PID" --log-file="$GALAXY_REPORTS_LOG" $@
+find_server $GALAXY_REPORTS_CONFIG
+$run_server $server_args

run_tool_shed.sh

@@ -1,29 +1,27 @@
 #!/bin/sh
 
-cd `dirname $0`
-
-: ${GALAXY_VIRTUAL_ENV:=.venv}
-
-if [ -d "$GALAXY_VIRTUAL_ENV" ];
-then
-    . "$GALAXY_VIRTUAL_ENV/bin/activate"
-fi
-
-./scripts/common_startup.sh
-
-: ${GALAXY_VIRTUAL_ENV:=.venv}
-
-if [ -d "$GALAXY_VIRTUAL_ENV" ];
-then
-    . "$GALAXY_VIRTUAL_ENV/bin/activate"
-fi
+cd "$(dirname "$0")"
 
 
-tool_shed=`./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $@`
-args=$@
+TOOL_SHED_PID=${TOOL_SHED_PID:-tool_shed_webapp.pid}
+TOOL_SHED_LOG=${TOOL_SHED_LOG:-tool_shed_webapp.log}
+PID_FILE=$TOOL_SHED_PID
+LOG_FILE=$TOOL_SHED_LOG
+
+. ./scripts/common_startup_functions.sh
+
+parse_common_args $@
+
+run_common_start_up
+
+setup_python
+
+
+tool_shed=`./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $parser_args`
+args=$parser_args
 
 if [ $? -eq 0 ] ; then
-	bash ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_tool_shed.sh $@
+	bash ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_tool_shed.sh $parser_args
 	args=`echo $@ | sed "s#-\?-bootstrap_from_tool_shed $tool_shed##"`
 fi
 
@@ -32,10 +30,13 @@ if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then
         TOOL_SHED_CONFIG_FILE=tool_shed_wsgi.ini
     elif [ -f config/tool_shed.ini ]; then
         TOOL_SHED_CONFIG_FILE=config/tool_shed.ini
+    elif [ -f config/tool_shed.yml ]; then
+        TOOL_SHED_CONFIG_FILE=config/tool_shed.yml
     else
-        TOOL_SHED_CONFIG_FILE=config/tool_shed.ini.sample
+        TOOL_SHED_CONFIG_FILE=config/tool_shed.yml.sample
     fi
     export TOOL_SHED_CONFIG_FILE
 fi
 
-python ./scripts/paster.py serve $TOOL_SHED_CONFIG_FILE --pid-file=tool_shed_webapp.pid --log-file=tool_shed_webapp.log $args
+find_server $TOOL_SHED_CONFIG_FILE
+$run_server $server_args

scripts/common_startup_functions.sh

@@ -0,0 +1,129 @@
+#!/bin/sh
+
+uwsgi_args="--master --pythonpath=lib"
+
+parse_common_args() {
+    INITIALIZE_TOOL_DEPENDENCIES=1
+    # Pop args meant for common_startup.sh
+    while :
+    do
+        case "$1" in
+            --skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip|--replace-pip)
+                common_startup_args="$common_startup_args $1"
+                shift
+                ;;
+            --skip-tool-dependency-initialization)
+                INITIALIZE_TOOL_DEPENDENCIES=0
+                shift
+                ;;
+            --skip-venv)
+                skip_venv=1
+                common_startup_args="$common_startup_args $1"
+                shift
+                ;;
+            --stop-daemon)
+                common_startup_args="$common_startup_args $1"
+                paster_args="$paster_args $1"
+                uwsgi_args="$uwsgi_args --stop $PID_FILE"
+                stop_daemon_arg_set=1
+                shift
+                ;;
+            --restart|restart)
+                if [ "$1" = "--restart" ]
+                then
+                    paster_args="$paster_args restart"
+                else
+                    paster_args="$paster_args $1"
+                fi
+                uwsgi_args="$uwsgi_args --reload $PID_FILE"
+                daemon_or_restart_arg_set=1
+                shift
+                ;;
+            --daemon)
+                paster_args="$paster_args $1"
+                # --daemonize2 waits until after the application has loaded
+                # to daemonize, thus it stops if any errors are found
+                uwsgi_args="$uwsgi_args --daemonize2 $LOG_FILE --safe-pidfile $PID_FILE"
+                daemon_or_restart_arg_set=1
+                shift
+                ;;
+            --wait)
+                wait_arg_set=1
+                shift
+                ;;
+            "")
+                break
+                ;;
+            *)
+                paster_args="$paster_args $1"
+                uwsgi_args="$uwsgi_args $1"
+                shift
+                ;;
+        esac
+    done
+}
+
+run_common_start_up() {
+    ./scripts/common_startup.sh $common_startup_args || exit 1
+}
+
+setup_python() {
+    # If there is a .venv/ directory, assume it contains a virtualenv that we
+    # should run this instance in.
+    GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-.venv}"
+    if [ -d "$GALAXY_VIRTUAL_ENV" -a -z "$skip_venv" ];
+    then
+        [ -n "$PYTHONPATH" ] && { echo 'Unsetting $PYTHONPATH'; unset PYTHONPATH; }
+        echo "Activating virtualenv at $GALAXY_VIRTUAL_ENV"
+        . "$GALAXY_VIRTUAL_ENV/bin/activate"
+    fi
+
+    # If you are using --skip-venv we assume you know what you are doing but warn
+    # in case you don't.
+    [ -n "$PYTHONPATH" ] && echo 'WARNING: $PYTHONPATH is set, this can cause problems importing Galaxy dependencies'
+
+    python ./scripts/check_python.py || exit 1
+}
+
+find_uwsgi() {
+    # Look for uwsgi
+    if [ -z "$skip_venv" -a -x $GALAXY_VIRTUAL_ENV/bin/uwsgi ]; then
+        UWSGI=$GALAXY_VIRTUAL_ENV/bin/uwsgi
+    elif command -v uwsgi >/dev/null 2>&1; then
+        UWSGI=uwsgi
+    else
+        echo 'ERROR: Could not find uwsgi executable'
+        exit 1
+    fi
+}
+
+find_server() {
+    server_config="$1"
+    server_config_style="ini-paste"
+    default_webserver="paste"
+    case "$server_config" in
+        *.y*ml*)
+            server_config_style="yaml"
+            default_webserver="uwsgi"  # paste incapable of this
+            ;;
+    esac
+
+    APP_WEBSERVER=${APP_WEBSERVER:-$default_webserver}
+    if [ "$APP_WEBSERVER" = "uwsgi" ];
+    then
+        # Look for uwsgi
+        if [ -z "$skip_venv" -a -x $GALAXY_VIRTUAL_ENV/bin/uwsgi ]; then
+            UWSGI=$GALAXY_VIRTUAL_ENV/bin/uwsgi
+        elif command -v uwsgi >/dev/null 2>&1; then
+            UWSGI=uwsgi
+        else
+            echo 'ERROR: Could not find uwsgi executable'
+            exit 1
+        fi
+        run_server="$UWSGI"
+        server_args="--$server_config_style $server_config $uwsgi_args"
+    else
+        run_server="python"
+        server_args="./scripts/paster.py serve $server_config $paster_args --pid-file $PID_FILE --log-file $LOG_FILE $paster_args"
+    fi
+}

scripts/config_sample_to_kwalify.py

@@ -0,0 +1,71 @@
+from __future__ import print_function
+
+import sys
+
+
+def main():
+    """Entry point for conversion procedure."""
+    found_app_main = False
+    current_section_desc = []
+    current_section_options = []
+    try:
+        sample = sys.argv[1]
+    except KeyError:
+        sample = "config/galaxy.ini.sample"
+
+    for line in open(sample, "r"):
+        is_app_main = line.startswith('[app:main]')
+        if not found_app_main and not is_app_main:
+            continue
+        if is_app_main:
+            found_app_main = True
+            continue
+
+        line = line.strip()
+        if not line.startswith("#"):
+            for section_option in current_section_options:
+                _dump_option(section_option, current_section_desc)
+            current_section_desc = []
+            current_section_options = []
+        if line.startswith("[galaxy_amqp]"):
+            break
+
+        if line.startswith("# ") or "#" == line:
+            current_section_desc.append(line[2:])
+        elif line.startswith("#"):
+            current_section_options.append(line)
+
+
+def _dump_option(option, current_section_desc):
+    def print_line(line):
+        print((" " * 6) + line)
+    if "=" not in option:
+        print(option)
+    key, default = [s.strip() for s in option.split("=", 1)]
+    key = key[1:]  # strip #
+    if default.strip().lower() in ["true", "false"]:
+        default = default.lower()
+        type = "bool"
+    elif default.isdigit():
+        type = "int"
+    else:
+        try:
+            float(default)
+            type = "float"
+        except ValueError:
+            type = "str"
+    if default == "None":
+        default = None
+    print_line("%s:" % key)
+    print_line("  type: %s" % type)
+    if default is not None:
+        print_line("  default: %s" % default)
+    # print_line("  required: false")
+    print_line("  desc: |")
+    for line in current_section_desc:
+        print_line("    %s" % line)
+    print_line("")
+
+
+if __name__ == "__main__":
+    main()

scripts/manage_tools.py

@@ -18,7 +18,7 @@ if not os.path.exists( config_file ):
     sys.exit( 1 )
 repo = 'lib/tool_shed/galaxy_install/migrate'
 
-properties = load_app_properties( ini_file=config_file )
+properties = load_app_properties( config_file=config_file )
 cp = SafeConfigParser()
 cp.read( config_file )
 

test/unit/config/1607_root_filters/config/tool_shed.ini

@@ -0,0 +1,188 @@
+# ---- HTTP Server ----------------------------------------------------------
+
+[server:main]
+
+use = egg:Paste#http
+port = 9009
+
+# The address on which to listen. By default, only listen to localhost
+# (the Tool Shed will not be accessible over the network).
+# Use '0.0.0.0' to listen on all available network interfaces.
+#host = 0.0.0.0
+host = 127.0.0.1
+
+use_threadpool = true
+threadpool_workers = 10
+# Set the number of seconds a thread can work before you should kill it
+# (assuming it will never finish) to 3 hours.
+threadpool_kill_thread_limit = 10800
+
+# Define the proxy-prefix filter.
+[filter:proxy-prefix]
+use = egg:PasteDeploy#prefix
+prefix = /shed
+
+# ---- Galaxy Tool Shed -----------------------------------------------------
+
+[app:main]
+
+filter-with = proxy-prefix
+
+# Specifies the factory for the universe WSGI application
+paste.app_factory = galaxy.webapps.tool_shed.buildapp:app_factory
+
+# Verbosity of console log messages.  Acceptable values can be found here:
+# https://docs.python.org/2/library/logging.html#logging-levels
+#log_level = DEBUG
+
+# By default, the Tool Shed uses a SQLite database at 'database/community.sqlite'.  You
+# may use a SQLAlchemy connection string to specify an external database
+# instead.  This string takes many options which are explained in detail in the
+# config file documentation.
+#database_connection = sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE
+
+# Where the hgweb.config file is stored.
+# The default is the Galaxy installation directory.
+#hgweb_config_dir = None
+
+# Where tool shed repositories are stored.
+#file_path = database/community_files
+
+# Temporary storage for additional datasets,
+# this should be shared through the cluster
+#new_file_path = database/tmp
+
+# File containing old-style genome builds
+#builds_file_path = tool-data/shared/ucsc/builds.txt
+
+# Format string used when showing date and time information.
+# The string may contain:
+# - the directives used by Python time.strftime() function (see
+#   https://docs.python.org/2/library/time.html#time.strftime ),
+# - $locale (complete format string for the server locale),
+# - $iso8601 (complete format string as specified by ISO 8601 international
+#   standard).
+#pretty_datetime_format = $locale (UTC)
+
+# -- Repository and Tool search
+# Using the script located at scripts/build_ts_whoosh_index.py
+# you can generate search index and allow full text API searching over
+# the repositories and tools within the Tool Shed given that you specify
+# the following two config options.
+#toolshed_search_on = True
+#whoosh_index_dir = database/toolshed_whoosh_indexes
+
+# The following boosts are used to customize this instance's TS search.
+# The higher the boost, the more importance the scoring algorithm gives to the
+# given field.
+
+# For searching repositories at /api/repositories:
+#repo_name_boost = 0.9
+#repo_description_boost = 0.6
+#repo_long_description_boost = 0.5
+#repo_homepage_url_boost = 0.3
+#repo_remote_repository_url_boost = 0.2
+#repo_owner_username_boost = 0.3
+
+# For searching tools at /api/tools
+#tool_name_boost = 1.2
+#tool_description_boost = 0.6
+#tool_help_boost = 0.4
+#tool_repo_owner_username = 0.3
+
+# -- Analytics
+
+# You can enter tracking code here to track visitor's behavior
+# through your Google Analytics account. Example: UA-XXXXXXXX-Y
+#ga_code = None
+
+# -- Users and Security
+
+# The Tool Shed encodes various internal values when these values will be output in
+# some format (for example, in a URL or cookie).  You should set a key to be
+# used by the algorithm that encodes and decodes these values.  It can be any
+# string.  If left unchanged, anyone could construct a cookie that would grant
+# them access to others' sessions.
+# One simple way to generate a value for this is with the shell command:
+#   python -c 'import time; print time.time()' | md5sum | cut -f 1 -d ' '
+#id_secret = changethisinproductiontoo
+
+# User authentication can be delegated to an upstream proxy server (usually
+# Apache).  The upstream proxy should set a REMOTE_USER header in the request.
+# Enabling remote user disables regular logins.  For more information, see:
+# https://wiki.galaxyproject.org/Admin/Config/ApacheProxy
+#use_remote_user = False
+
+# If use_remote_user is enabled, anyone who can log in to the Galaxy host may
+# impersonate any other user by simply sending the appropriate header. Thus a
+# secret shared between the upstream proxy server, and Galaxy is required.
+# If anyone other than the Galaxy user is using the server, then apache/nginx
+# should pass a value in the header 'GX_SECRET' that is identical the one below
+#remote_user_secret = changethisinproductiontoo
+
+# Configuration for debugging middleware
+#debug = False
+
+# Check for WSGI compliance.
+#use_lint = False
+
+# Intercept print statements and show them on the returned page.
+#use_printdebug = True
+
+# NEVER enable this on a public site (even test or QA)
+#use_interactive = true
+
+# Administrative users - set this to a comma-separated list of valid Tool Shed
+# users (email addresses).  These users will have access to the Admin section
+# of the server, and will have access to create users, groups, roles,
+# libraries, and more.
+#admin_users = None
+
+# Force everyone to log in (disable anonymous access)
+#require_login = False
+
+# For use by email messages sent from the tool shed
+#smtp_server = smtp.your_tool_shed_server
+#email_from = your_tool_shed_email@server
+
+# If your SMTP server requires a username and password, you can provide them
+# here (password in cleartext here, but if your server supports STARTTLS it
+# will be sent over the network encrypted).
+#smtp_username = None
+#smtp_password = None
+
+# If your SMTP server requires SSL from the beginning of the connection
+#smtp_ssl = False
+
+# The URL linked by the "Support" link in the "Help" menu.
+#support_url = https://wiki.galaxyproject.org/Support
+
+# Address to join mailing list
+#mailing_join_addr = galaxy-announce-join@bx.psu.edu
+
+# Write thread status periodically to 'heartbeat.log' (careful, uses disk
+#  space rapidly!)
+#use_heartbeat = True
+
+# Profiling middleware (cProfile based)
+#use_profile = True
+
+# Enable creation of Galaxy flavor Docker Image
+#enable_galaxy_flavor_docker_image = False
+
+# Show a message box under the masthead.
+#message_box_visible = False
+#message_box_content = None
+#message_box_class = info
+
+# Serving static files (needed if running standalone)
+#static_enabled = True
+#static_cache_time = 360
+#static_dir = static/
+#static_images_dir = static/images
+#static_favicon_dir = static/favicon.ico
+#static_scripts_dir = static/scripts/
+#static_style_dir = static/style/blue
+
+# Sentry (getsentry.com) DSN for catching bugs.
+#sentry_dsn = None

test/unit/config/1607_root_samples/config/reports.ini

@@ -39,16 +39,12 @@ prefix = /reports
 
 # Specifies the factory for the universe WSGI application
 paste.app_factory = galaxy.webapps.reports.buildapp:app_factory
-
-# Verbosity of console log messages.  Acceptable values can be found here:
-# https://docs.python.org/2/library/logging.html#logging-levels
-#log_level = DEBUG
+log_level = DEBUG
 
 # Database connection
-# Galaxy reports are intended for production Galaxy instances, so sqlite (and the default value
-# below) is not supported. An SQLAlchemy connection string should be used specify an external
-# database.
-#database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
+# Galaxy reports are intended for production Galaxy instances, so sqlite is not supported.
+# You may use a SQLAlchemy connection string to specify an external database.
+#database_connection = postgres:///galaxy_test?user=postgres&password=postgres
 
 # Where dataset files are saved
 #file_path = database/files
@@ -59,6 +55,13 @@ paste.app_factory = galaxy.webapps.reports.buildapp:app_factory
 # used for the cache
 #template_cache_path = database/compiled_templates/reports
 
+# Session support (beaker)
+use_beaker_session = True
+session_type = memory
+session_data_dir = %(here)s/database/beaker_sessions
+session_key = galaxysessions
+session_secret = changethisinproduction
+
 # Configuration for debugging middleware
 #debug = False
 
@@ -88,4 +91,4 @@ paste.app_factory = galaxy.webapps.reports.buildapp:app_factory
 # static_style_dir = %(here)s/static/june_2007_style/blue
 
 # Sentry (getsentry.com) DSN for catching bugs.
-#sentry_dsn = None
+#sentry_dsn = None

test/unit/config/1607_root_samples/config/tool_shed.ini

@@ -119,6 +119,9 @@ paste.app_factory = galaxy.webapps.tool_shed.buildapp:app_factory
 # Check for WSGI compliance.
 #use_lint = False
 
+# Intercept print statements and show them on the returned page.
+#use_printdebug = True
+
 # NEVER enable this on a public site (even test or QA)
 #use_interactive = true
 
@@ -175,4 +178,4 @@ paste.app_factory = galaxy.webapps.tool_shed.buildapp:app_factory
 #static_style_dir = static/style/blue
 
 # Sentry (getsentry.com) DSN for catching bugs.
-#sentry_dsn = None
+#sentry_dsn = None

test/unit/config/test_config_manage.py

@@ -0,0 +1,130 @@
+
+import os
+import shutil
+import tempfile
+
+import yaml
+
+from galaxy.webapps.config_manage import main
+
+THIS_DIR = os.path.dirname(__file__)
+
+
+def test_reports_conversion_1607_sample():
+    with _config_directory("1607_root_samples") as config_dir:
+        config_dir.manage_cli(["convert", "reports"])
+        config_dir.assert_not_exists("config/reports.ini")
+        config_dir.assert_is_yaml("config/reports.yml")
+        config_dir.assert_moved("config/reports.ini", "config/reports.ini.backup")
+        with config_dir.open("config/reports.yml") as f:
+            config = yaml.load(f)
+        assert "reports" in config
+        reports_config = config["reports"] or {}
+        assert "use_beaker_session" not in reports_config
+
+        assert "uwsgi" in config
+        uwsgi_config = config["uwsgi"]
+        assert uwsgi_config["module"] == "galaxy.webapps.reports.buildapp:uwsgi_app()"
+
+
+def test_reports_build_sample():
+    with _config_directory("1607_root_samples") as config_dir:
+        config_dir.assert_not_exists("config/reports.yml.sample")
+        config_dir.manage_cli(["build_sample_yaml", "reports", "--add-comments"])
+        config_dir.assert_is_yaml("config/reports.yml.sample")
+
+
+def test_shed_conversion_1607_sample():
+    with _config_directory("1607_root_samples") as config_dir:
+        config_dir.manage_cli(["convert", "tool_shed"])
+        config_dir.assert_not_exists("config/tool_shed.ini")
+        config_dir.assert_is_yaml("config/tool_shed.yml")
+        config_dir.assert_moved("config/tool_shed.ini", "config/tool_shed.ini.backup")
+
+
+def test_shed_build_sample():
+    with _config_directory("1607_root_samples") as config_dir:
+        config_dir.assert_not_exists("config/tool_shed.yml.sample")
+        config_dir.manage_cli(["build_sample_yaml", "tool_shed", "--add-comments"])
+        config_dir.assert_is_yaml("config/tool_shed.yml.sample")
+
+
+def test_shed_conversion_1607_prefix():
+    with _config_directory("1607_root_filters") as config_dir:
+        config_dir.manage_cli(["convert", "tool_shed"])
+        config_dir.assert_not_exists("config/tool_shed.ini")
+        config_dir.assert_is_yaml("config/tool_shed.yml")
+        config_dir.assert_moved("config/tool_shed.ini", "config/tool_shed.ini.backup")
+        with config_dir.open("config/tool_shed.yml") as f:
+            config = yaml.load(f)
+        assert "uwsgi" in config
+        uwsgi_config = config["uwsgi"]
+        assert "module" not in uwsgi_config
+        assert uwsgi_config["mount"].startswith("/shed=galaxy.")
+
+
+def test_reports_conversion_1607_gzip():
+    with _config_directory("1607_root_filters") as config_dir:
+        config_dir.manage_cli(["convert", "reports"])
+        config_dir.assert_not_exists("config/reports.ini")
+        config_dir.assert_is_yaml("config/reports.yml")
+        config_dir.assert_moved("config/reports.ini", "config/reports.ini.backup")
+        with config_dir.open("config/reports.yml") as f:
+            config = yaml.load(f)
+        assert "uwsgi" in config
+        uwsgi_config = config["uwsgi"]
+        assert uwsgi_config["http-auto-gzip"] is True, uwsgi_config
+
+
+def test_build_uwsgi_yaml():
+    with _config_directory("1607_root_samples") as config_dir:
+        config_dir.manage_cli(["build_uwsgi_yaml"])
+
+
+class _TestConfigDirectory(object):
+
+    def __init__(self, base_name):
+        temp_directory = tempfile.mkdtemp()
+        os.removedirs(temp_directory)
+        source_dir = os.path.join(THIS_DIR, base_name)
+        shutil.copytree(source_dir, temp_directory)
+        self.source_dir = source_dir
+        self.temp_directory = temp_directory
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, tb):
+        shutil.rmtree(self.temp_directory)
+
+    def assert_exists(self, path):
+        assert os.path.exists(os.path.join(self.temp_directory, path))
+
+    def assert_not_exists(self, path):
+        assert not os.path.exists(os.path.join(self.temp_directory, path))
+
+    def assert_is_yaml(self, path):
+        self.assert_exists(path)
+        with self.open(path, "r") as f:
+            return yaml.load(f)
+
+    def open(self, path, *args):
+        return open(os.path.join(self.temp_directory, path), *args)
+
+    def assert_moved(self, from_path, to_path):
+        with open(os.path.join(self.source_dir, from_path), "r") as f:
+            source_contents = f.read()
+
+        with self.open(to_path, "r") as f:
+            dest_contents = f.read()
+
+        assert source_contents == dest_contents
+
+    def manage_cli(self, argv):
+        real_argv = ["--galaxy_root", self.temp_directory]
+        real_argv.extend(argv)
+        main(real_argv)
+
+
+def _config_directory(base_name):
+    return _TestConfigDirectory(base_name)