From b099ea34cf3a5721b327da72a0386cf031fac930 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 8 Jul 2019 18:14:53 +0200 Subject: [PATCH 01/83] Update galaxy config to include config_dir and data_dir options --- doc/source/admin/galaxy_options.rst | 54 ++++++++++++++++++--- lib/galaxy/config/sample/galaxy.yml.sample | 37 ++++++++++---- lib/galaxy/webapps/galaxy/config_schema.yml | 19 ++++++++ 3 files changed, 94 insertions(+), 16 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 30f04e85099..47818ceb792 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1,3 +1,28 @@ +~~~~~~~~~~~~~~ +``config_dir`` +~~~~~~~~~~~~~~ + +:Description: + The directory that will be prepended to relative paths in options + specifying other Galaxy config files (e.g. datatypes_config_file). + Defaults to the directory in which galaxy.yml is located. +:Default: ``false`` +:Type: str + + +~~~~~~~~~~~~ +``data_dir`` +~~~~~~~~~~~~ + +:Description: + The directory that will be prepended to relative paths in options + specifying Galaxy data/cache directories and files (such as the + default SQLite database, file_path, etc.). Defaults to `database/` + if running Galaxy from source or `/data` otherwise. +:Default: ``false`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~ ``database_connection`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -271,14 +296,13 @@ ~~~~~~~~~~~~~~~~~~~~~~~ :Description: - Path to the directory in which tool dependencies are placed. This - is used by the Tool Shed to install dependencies and can also be - used by administrators to manually install or link to - dependencies. For details, see: - https://galaxyproject.org/admin/config/tool-dependencies Set the + Various dependency resolver configuration parameters will have + defaults set relative to this path, such as the default conda + prefix, default Galaxy packages path, legacy tool shed + dependencies path, and the dependency cache directory. Set the string to None to explicitly disable tool dependency handling. If this option is set to none or an invalid path, installing tools - with dependencies from the Tool Shed will fail. + with dependencies from the Tool Shed or in Conda will fail. :Default: ``database/dependencies`` :Type: str @@ -405,7 +429,9 @@ share. Set the following option to True to cache the dependencies in a folder. This option is beta and should only be used if you experience long waiting times before a job is actually submitted - to your cluster. + to your cluster. This only affects tools where some requirements + can be resolved but not others, most modern best practice tools + can use prebuilt environments in the Conda directory. :Default: ``false`` :Type: bool @@ -3210,6 +3236,20 @@ :Type: seq +~~~~~~~~~~~~~~~~~~~~~~~~~ +``dependency_resolution`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative representation of various dependency resolution + parameters. Takes the dictified version of a DependencyManager + object - so this is ideal for automating the configuration of + dependency resolution from one application that uses a + DependencyManager to another. +:Default: ``None`` +:Type: map + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``default_job_resubmission_condition`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 5b38d4832d6..741bd672f11 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -98,6 +98,17 @@ uwsgi: galaxy: + # The directory that will be prepended to relative paths in options + # specifying other Galaxy config files (e.g. datatypes_config_file). + # Defaults to the directory in which galaxy.yml is located. + #config_dir: false + + # The directory that will be prepended to relative paths in options + # specifying Galaxy data/cache directories and files (such as the + # default SQLite database, file_path, etc.). Defaults to `database/` + # if running Galaxy from source or `/data` otherwise. + #data_dir: false + # By default, Galaxy uses a SQLite database at # 'database/universe.sqlite'. You may use a SQLAlchemy connection # string to specify an external database instead. This string takes @@ -215,14 +226,13 @@ galaxy: # an attribute in the tag. #tool_path: tools - # Path to the directory in which tool dependencies are placed. This - # is used by the Tool Shed to install dependencies and can also be - # used by administrators to manually install or link to dependencies. - # For details, see: https://galaxyproject.org/admin/config/tool- - # dependencies Set the string to None to explicitly disable tool - # dependency handling. If this option is set to none or an invalid - # path, installing tools with dependencies from the Tool Shed will - # fail. + # Various dependency resolver configuration parameters will have + # defaults set relative to this path, such as the default conda + # prefix, default Galaxy packages path, legacy tool shed dependencies + # path, and the dependency cache directory. Set the string to None to + # explicitly disable tool dependency handling. If this option is set + # to none or an invalid path, installing tools with dependencies from + # the Tool Shed or in Conda will fail. #tool_dependency_dir: database/dependencies # The dependency resolvers config file specifies an ordering and @@ -277,7 +287,9 @@ galaxy: # share. Set the following option to True to cache the dependencies # in a folder. This option is beta and should only be used if you # experience long waiting times before a job is actually submitted to - # your cluster. + # your cluster. This only affects tools where some requirements can + # be resolved but not others, most modern best practice tools can use + # prebuilt environments in the Conda directory. #use_cached_dependency_manager: false # By default the tool_dependency_cache_dir is the _cache directory of @@ -1540,6 +1552,13 @@ galaxy: # dependency_resolvers_config_file is used. #dependency_resolvers: null + # Alternative representation of various dependency resolution + # parameters. Takes the dictified version of a DependencyManager + # object - so this is ideal for automating the configuration of + # dependency resolution from one application that uses a + # DependencyManager to another. + #dependency_resolution: null + # When jobs fail due to job runner problems, Galaxy can be configured # to retry these or reroute the jobs to new destinations. Very fine # control of this is available with resubmit declarations in diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 0a586fd22be..f63becc1624 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -26,6 +26,25 @@ mapping: required: true mapping: + config_dir: + type: str + default: false + required: false + desc: | + The directory that will be prepended to relative paths in options specifying + other Galaxy config files (e.g. datatypes_config_file). Defaults to the + directory in which galaxy.yml is located. + + data_dir: + type: str + default: false + required: false + desc: | + The directory that will be prepended to relative paths in options specifying + Galaxy data/cache directories and files (such as the default SQLite database, + file_path, etc.). Defaults to `database/` if running Galaxy from source or + `/data` otherwise. + database_connection: type: str default: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE From a67ad5f8a43ca095c6256b71c00e48a281a0e772 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 8 Jul 2019 18:15:58 +0200 Subject: [PATCH 02/83] Update the galaxy-config script for YAML configs --- lib/galaxy/config/script.py | 133 +++++++++++++++++------------------- 1 file changed, 62 insertions(+), 71 deletions(-) diff --git a/lib/galaxy/config/script.py b/lib/galaxy/config/script.py index 6cf211a890d..2efc479aa6c 100644 --- a/lib/galaxy/config/script.py +++ b/lib/galaxy/config/script.py @@ -12,76 +12,69 @@ except ImportError: pip = None -CONFIGURE_URL = "https://wiki.galaxyproject.org/Admin/Config/Performance/ProductionServer" +CONFIGURE_URL = "https://docs.galaxyproject.org/en/master/admin/" DESCRIPTION = "Initialize a directory with a minimal Galaxy config." HELP_CONFIG_DIR = "Directory containing the configuration files for Galaxy." HELP_DATA_DIR = "Directory containing Galaxy-created data." HELP_FORCE = "Overwrite existing files if they already exist." -HELP_WSGI_SERVER = ("Web server stack used to host Galaxy web application, and " - "if uWSGI, which protocol to use.") -HELP_LIBDRMAA = ("Configure Galaxy to submit jobs to a cluster via DRMAA by " - "supplying the path to a libdrmaa.so file using this argument.") -HELP_INSTALL = ("Install optional dependencies required by specified configuration " - "(e.g. drmaa, uwsgi, etc...).") -HELP_HOST = ('Host to bind Galaxy to - defaults to localhost. Specify an IP ' - 'address or "all" to listen on all interfaces.') +HELP_WSGI_SERVER = ("Web server stack used to host Galaxy web application, and if uWSGI, which protocol to use.") +HELP_LIBDRMAA = ( + "Configure Galaxy to submit jobs to a cluster via DRMAA by supplying the path to a libdrmaa.so file using this " + "argument." +) +HELP_INSTALL = ("Install optional dependencies required by specified configuration (e.g. drmaa, etc...).") +HELP_HOST = ( + 'Host to bind Galaxy to - defaults to localhost. Specify an IP address or "all" to listen on all interfaces.' +) HELP_PORT = ("Port to bind Galaxy to.") -HELP_DB_CONN = ('Galaxy database connection URI.') +HELP_DB_CONN = ("Galaxy database connection URI.") DEFAULT_HOST = "localhost" -DEFAULT_INI = "galaxy.ini" +DEFAULT_YML = "galaxy.yml" DEFAULT_DB_CONN = 'sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE' SAMPLES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample')) -GALAXY_CONFIG_TEMPLATE_FILE = os.path.join(SAMPLES_PATH, 'galaxy.ini.sample') +GALAXY_CONFIG_TEMPLATE_FILE = os.path.join(SAMPLES_PATH, 'galaxy.yml.sample') STATIC_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'web', 'framework', 'static')) -# The sample is used as the default config file for Galaxy started without a -# config, and we don't want to duplicate the whole thing into galaxy.config for -# templating, so for now just substitute some lines. In the future we will -# build configs differently. +MSG_CONFIG_SUMMARY = """ +For help on configuring Galaxy, consult the documentation at: \n {} + +Additional sample configuration files for various Galaxy components (jobs, +datatypes, etc.) can be found in:\n {} + +Start Galaxy by running the command from directory [{}]: +""" + +# The sample is used as the default config file for Galaxy started without a config, and we don't want to duplicate the +# whole thing into galaxy.config for templating, so for now just substitute some lines. In the future we will build +# configs differently. GALAXY_CONFIG_SUBSTITUTIONS = { - '#port = 8080': 'port = ${port}', - '#host = 127.0.0.1': 'host = ${host}', - 'http = 127.0.0.1:8080': '${uwsgi_transport} = ${host}:${port}', - 'static-map = /static=/path/to/galaxy/web/framework/static': 'static-map = /static=${static_path}', - 'static-map = /static/style=/path/to/galaxy/web/framework/static/style/blue': 'static-map = /static/style=${static_path}/style/blue', - '#config_dir = None': 'config_dir = ${config_dir}', - '#data_dir = None': 'data_dir = ${data_dir}', - '#database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE': 'database_connection = ${database_connection}', + ' http: 127.0.0.1:8080': ' ${uwsgi_transport}: ${host}:${port}', + ' static-map: /static/style=static/style/blue': ' static-map: /static=${static_path}/style/blue', + ' static-map: /static=static': ' static-map: /static=${static_path}', + ' static-map: /favicon.ico=static/favicon.ico': ' static-map: /static=${static_path}/favicon.ico', + ' virtualenv: .venv': ' #venv: .venv # not used when running installed', + ' pythonpath: lib': ' #pythonpath: lib # not used when running installed', + ' #config_dir: false': ' config_dir: ${config_dir}', + ' #data_dir: false': ' data_dir: ${data_dir}', + ' #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE': ' database_connection: ${database_connection}', } def main(argv=None): dependencies = [] arg_parser = ArgumentParser(description=DESCRIPTION) - arg_parser.add_argument("--config-dir", - default=".", - help=HELP_CONFIG_DIR) - arg_parser.add_argument("--data-dir", - default="./data", - help=HELP_DATA_DIR) - arg_parser.add_argument("--wsgi-server", - choices=["paster", "uwsgi-http", "uwsgi-native"], - default="paster", + arg_parser.add_argument("--config-dir", default=".", help=HELP_CONFIG_DIR) + arg_parser.add_argument("--data-dir", default="./data", help=HELP_DATA_DIR) + arg_parser.add_argument("--wsgi-server", choices=["uwsgi-http", "uwsgi-native"], default="uwsgi-http", help=HELP_WSGI_SERVER) - arg_parser.add_argument("--host", - default=DEFAULT_HOST, - help=HELP_HOST) - arg_parser.add_argument("--port", - default="8080", - help=HELP_PORT) - arg_parser.add_argument("--db-conn", - default=DEFAULT_DB_CONN, - help=HELP_DB_CONN) - arg_parser.add_argument("--install", - action="store_true", - help=HELP_INSTALL) - arg_parser.add_argument("--force", - action="store_true", - default=False, - help=HELP_FORCE) + arg_parser.add_argument("--host", default=DEFAULT_HOST, help=HELP_HOST) + arg_parser.add_argument("--port", default="8080", help=HELP_PORT) + arg_parser.add_argument("--db-conn", default=DEFAULT_DB_CONN, help=HELP_DB_CONN) + arg_parser.add_argument("--install", action="store_true", help=HELP_INSTALL) + arg_parser.add_argument("--force", action="store_true", default=False, help=HELP_FORCE) args = arg_parser.parse_args(argv) config_dir = args.config_dir relative_config_dir = config_dir @@ -90,33 +83,27 @@ def main(argv=None): data_dir = os.path.abspath(data_dir) mode = _determine_mode(args) - if mode.startswith("uwsgi-"): - dependencies.append("uwsgi") + if args.db_conn.startswith("postgresql://"): + dependencies.append("psycopg2-binary") for directory in (config_dir, data_dir): if not os.path.exists(directory): os.makedirs(directory) print("Bootstrapping Galaxy configuration into directory %s" % relative_config_dir) - _handle_galaxy_ini(args, config_dir, data_dir) + _handle_galaxy_yml(args, config_dir, data_dir) _handle_install(args, dependencies) _print_config_summary(args, mode, relative_config_dir) def _print_config_summary(args, mode, relative_config_dir): - _print_galaxy_ini_info(args, mode) - print("") - print("For help on configuring Galaxy, consult the documentation at:\n ", CONFIGURE_URL) - print("") - print("Additional sample configuration files for various Galaxy components (jobs,") - print("datatypes, etc.) can be found in:\n ", SAMPLES_PATH) - print("") - print("Start Galaxy by running the command from directory [%s]:" % relative_config_dir) + _print_galaxy_yml_info(args, mode) + print(MSG_CONFIG_SUMMARY.format(CONFIGURE_URL, SAMPLES_PATH, relative_config_dir)) _print_galaxy_run(mode) -def _print_galaxy_ini_info(args, mode): - print(" - galaxy.ini created, update to configure Galaxy.") +def _print_galaxy_yml_info(args, mode): + print(" - galaxy.yml created, update to configure Galaxy.") print(" * Target web server %s" % mode) if args.host == DEFAULT_HOST: print(" * Binding to host localhost, remote clients will not be able to connect.") @@ -128,9 +115,9 @@ def _print_galaxy_ini_info(args, mode): def _print_galaxy_run(mode): if mode.startswith("uwsgi"): - print(" uwsgi --ini-paste galaxy.ini") + print(" uwsgi --yaml galaxy.yml") else: - print(" galaxy-paster serve galaxy.ini") + raise Exception("Unknown mode: %s" % mode) def _determine_mode(args): @@ -145,14 +132,14 @@ def _determine_host(args): return '0.0.0.0' if args.host == 'all' else args.host -def _determine_ini_file(config_dir): - return os.path.join(config_dir, DEFAULT_INI) +def _determine_yml_file(config_dir): + return os.path.join(config_dir, DEFAULT_YML) -def _handle_galaxy_ini(args, config_dir, data_dir): +def _handle_galaxy_yml(args, config_dir, data_dir): force = args.force - ini_file = _determine_ini_file(config_dir) - _check_file(ini_file, force) + yml_file = _determine_yml_file(config_dir) + _check_file(yml_file, force) uwsgi_transport = 'socket' if args.wsgi_server == 'uwsgi-native' else 'http' config_dict = dict( port=args.port, @@ -176,13 +163,13 @@ def _handle_galaxy_ini(args, config_dir, data_dir): galaxy_config = galaxy_config_template.safe_substitute( **config_dict ) - open(ini_file, "w").write(galaxy_config) + open(yml_file, "w").write(galaxy_config) def _handle_install(args, dependencies): if args.install and dependencies: if pip is None: - raise ImportError("Bootstrapping Pulsar dependencies requires pip library.") + raise ImportError("Bootstrapping Galaxy dependencies requires pip.") pip.main(["install"] + dependencies) @@ -191,3 +178,7 @@ def _check_file(path, force): if os.path.exists(path) and not force: print("File %s exists, exiting. Run with --force to replace configuration." % path, file=sys.stderr) sys.exit(1) + + +if __name__ == '__main__': + main() From 3b0494192570e0d0799eb0f99d0cba3196d383bf Mon Sep 17 00:00:00 2001 From: will Date: Mon, 3 Jun 2019 15:42:47 -0400 Subject: [PATCH 03/83] change # to javascript:void(0) so that the screen reader does not interpret the links as already visited --- .../galaxy/scripts/mvc/workflow/workflow-view.js | 6 +++--- templates/webapps/galaxy/workflow/editor.mako | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/galaxy/scripts/mvc/workflow/workflow-view.js b/client/galaxy/scripts/mvc/workflow/workflow-view.js index f4a29b16cf1..ec7a1d0bd7d 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-view.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-view.js @@ -453,7 +453,7 @@ export default Backbone.View.extend({ // Tool menu $("div.toolSectionBody").hide(); - $("div.toolSectionTitle > span").wrap(""); + $("div.toolSectionTitle > span").wrap(""); var last_expanded = null; $("div.toolSectionTitle").each(function() { var body = $(this).next("div.toolSectionBody"); @@ -494,7 +494,7 @@ export default Backbone.View.extend({ var $section = $( '
' + '
' + - 'Workflows' + + 'Workflows' + "
" + '
' + '
' + @@ -530,7 +530,7 @@ export default Backbone.View.extend({ } }); var $add = $("") - .attr("href", "#") + .attr("href", "javascript:void(0)") .html(workflow.name) .on("click", () => { self.add_node_for_subworkflow(workflow.latest_id, workflow.name); diff --git a/templates/webapps/galaxy/workflow/editor.mako b/templates/webapps/galaxy/workflow/editor.mako index 187f634ca99..cfd0be69b9d 100644 --- a/templates/webapps/galaxy/workflow/editor.mako +++ b/templates/webapps/galaxy/workflow/editor.mako @@ -80,9 +80,9 @@ %if "[[" in tool.description and "]]" in tool.description: ${tool.description.replace( '[[', '' % tool.id ).replace( "]]", "" )} %elif tool.name: - ${tool.name | h} ${tool.description} + ${tool.name | h} ${tool.description} %else: - ${tool.description} + ${tool.description} %endif
%else: @@ -126,7 +126,7 @@
%for module in module_section["modules"]:
- + ${module['description']}
@@ -232,13 +232,13 @@
- + - + - +
@@ -287,7 +287,7 @@
- + ## Workflow tags. <%namespace file="/tagging_common.mako" import="render_individual_tagging_element" />
From 84ff9f35e135cea0b6d965024dfd9f178e4698f9 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 4 Jun 2019 14:37:20 -0400 Subject: [PATCH 04/83] Add aria labels and roles screen readers --- client/galaxy/scripts/layout/masthead.js | 4 ++-- client/galaxy/scripts/mvc/tool/tools.js | 2 +- .../scripts/mvc/workflow/workflow-view.js | 17 +++++++++++++---- templates/webapps/galaxy/workflow/editor.mako | 14 +++++++------- templates/webapps/tool_shed/base_panels.mako | 4 ++-- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/client/galaxy/scripts/layout/masthead.js b/client/galaxy/scripts/layout/masthead.js index dc41a0b738d..367f280f814 100644 --- a/client/galaxy/scripts/layout/masthead.js +++ b/client/galaxy/scripts/layout/masthead.js @@ -98,8 +98,8 @@ const View = Backbone.View.extend({ _template: function() { return `
diff --git a/client/galaxy/docs/galaxy-buttons.md b/client/galaxy/docs/galaxy-buttons.md index b959cb7ce60..30675603809 100644 --- a/client/galaxy/docs/galaxy-buttons.md +++ b/client/galaxy/docs/galaxy-buttons.md @@ -5,21 +5,21 @@ ``` ```vue -An anchor with .action-button +An anchor with .action-button ``` All the crazy permutations of menu button... ```vue -An anchor with .menu-button +An anchor with .menu-button ``` ```vue -An anchor with .menu-button.popup +An anchor with .menu-button.popup ``` ```vue - + An anchor with .menu-button.popup.split ``` diff --git a/client/galaxy/scripts/components/Tags/StatelessTags.vue b/client/galaxy/scripts/components/Tags/StatelessTags.vue index c45ab737cdd..4f8be8b4697 100644 --- a/client/galaxy/scripts/components/Tags/StatelessTags.vue +++ b/client/galaxy/scripts/components/Tags/StatelessTags.vue @@ -4,7 +4,7 @@ upstream component or environment that is accessed through props and events -->