mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge branch 'dev' into admin_grid_groups
This commit is contained in:
guerler
@@ -10,6 +10,7 @@ MY_UPSTREAM:=origin
|
||||
VENV?=.venv
|
||||
# Source virtualenv to execute command (flake8, sphinx, twine, etc...)
|
||||
IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi;
|
||||
CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/webapps/config_manage.py
|
||||
PROJECT_URL?=https://github.com/galaxyproject/galaxy
|
||||
GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01
|
||||
GRUNT_EXEC?=node_modules/grunt-cli/bin/grunt
|
||||
@@ -52,6 +53,42 @@ open-project: ## open project on github
|
||||
lint: ## check style using tox and flake8 for Python 2 and Python 3
|
||||
$(IN_VENV) tox -e py27-lint && tox -e py34-lint
|
||||
|
||||
uwsgi-rebuild-validation: ## rebuild uwsgi_config.yml kwalify schema against latest uwsgi master.
|
||||
$(CONFIG_MANAGE) build_uwsgi_yaml
|
||||
|
||||
tool-shed-config-validate: ## validate tool shed YAML configuration file
|
||||
$(CONFIG_MANAGE) validate tool_shed
|
||||
|
||||
tool-shed-config-lint: ## lint tool shed YAML configuration file
|
||||
$(CONFIG_MANAGE) lint tool_shed
|
||||
|
||||
tool-shed-config-convert-dry-run: ## convert old style tool shed ini to yaml (dry run)
|
||||
$(CONFIG_MANAGE) convert tool_shed --dry-run
|
||||
|
||||
tool-shed-config-convert: ## convert old style tool shed ini to yaml
|
||||
$(CONFIG_MANAGE) convert tool_shed
|
||||
|
||||
tool-shed-config-rebuild-sample: ## Rebuild sample tool shed yaml file from schema
|
||||
$(CONFIG_MANAGE) build_sample_yaml tool_shed --add-comments
|
||||
|
||||
reports-config-validate: ## validate reports YAML configuration file
|
||||
$(CONFIG_MANAGE) validate reports
|
||||
|
||||
reports-config-convert-dry-run: ## convert old style reports ini to yaml (dry run)
|
||||
$(CONFIG_MANAGE) convert reports --dry-run
|
||||
|
||||
reports-config-convert: ## convert old style reports ini to yaml
|
||||
$(CONFIG_MANAGE) convert reports
|
||||
|
||||
reports-config-rebuild-sample: ## Rebuild sample reports yaml file from schema
|
||||
$(CONFIG_MANAGE) build_sample_yaml reports --add-comments
|
||||
|
||||
reports-config-lint: ## lint reports YAML configuration file
|
||||
$(CONFIG_MANAGE) lint reports
|
||||
|
||||
reports-config-rebuild-rst: ## Rebuild sample reports RST docs
|
||||
$(CONFIG_MANAGE) build_rst reports > doc/source/admin/reports_options.rst
|
||||
|
||||
release-ensure-upstream: ## Ensure upstream branch for release commands setup
|
||||
ifeq (shell git remote -v | grep $(RELEASE_UPSTREAM), )
|
||||
git remote add $(RELEASE_UPSTREAM) git@github.com:galaxyproject/galaxy.git
|
||||
|
||||
@@ -162,7 +162,11 @@ var FolderToolbarView = Backbone.View.extend({
|
||||
if (this.validate_new_folder(folderDetails)){
|
||||
var folder = new mod_library_model.FolderAsModel();
|
||||
url_items = Backbone.history.fragment.split('/');
|
||||
current_folder_id = url_items[url_items.length-1];
|
||||
if(url_items.indexOf('page') > -1){
|
||||
current_folder_id = url_items[url_items.length-3];
|
||||
}else {
|
||||
current_folder_id = url_items[url_items.length-1];
|
||||
}
|
||||
folder.url = folder.urlRoot + current_folder_id ;
|
||||
|
||||
folder.save(folderDetails, {
|
||||
|
||||
@@ -384,6 +384,7 @@
|
||||
<datatype extension="rdf" type="galaxy.datatypes.triples:Rdf" display_in_upload="true"/>
|
||||
<datatype extension="jsonld" type="galaxy.datatypes.triples:Jsonld" display_in_upload="true"/>
|
||||
<!-- Excel datatypes -->
|
||||
<datatype extension="excel.xls" type="galaxy.datatypes.binary:ExcelXls" display_in_upload="true"/>
|
||||
<datatype extension="xlsx" type="galaxy.datatypes.binary:Xlsx" display_in_upload="true" />
|
||||
<datatype extension="btwisted" type="galaxy.datatypes.data:Text" subclass="true"/>
|
||||
<datatype extension="cai" type="galaxy.datatypes.data:Text" subclass="true"/>
|
||||
@@ -551,8 +552,12 @@
|
||||
</datatype>
|
||||
<datatype extension="fps" type="galaxy.datatypes.molecules:FPS" mimetype="text/html" display_in_upload="true" />
|
||||
<datatype extension="obfs" type="galaxy.datatypes.molecules:OBFS" mimetype="text/html" display_in_upload="true" />
|
||||
<datatype extension="drf" type="galaxy.datatypes.molecules:DRF" display_in_upload="true" />
|
||||
<datatype extension="phar" type="galaxy.datatypes.molecules:PHAR" display_in_upload="false" />
|
||||
<datatype extension="pdb" type="galaxy.datatypes.molecules:PDB" display_in_upload="true" />
|
||||
<datatype extension="pdbqt" type="galaxy.datatypes.molecules:PDBQT" display_in_upload="true" />
|
||||
<datatype extension="grd" type="galaxy.datatypes.molecules:grd" display_in_upload="true" />
|
||||
<datatype extension="grd.tgz" type="galaxy.datatypes.molecules:grdtgz" display_in_upload="true" />
|
||||
<!-- mothur formats -->
|
||||
<datatype extension="mothur.otu" type="galaxy.datatypes.mothur:Otu" display_in_upload="true"/>
|
||||
<datatype extension="mothur.list" type="galaxy.datatypes.mothur:Otu" subclass="true" display_in_upload="true"/>
|
||||
|
||||
@@ -908,12 +908,14 @@ use_interactive = True
|
||||
# sub-directories of files contained in their directory.
|
||||
#user_library_import_dir = None
|
||||
|
||||
# Add an option to the admin library upload tool allowing admins to paste
|
||||
# Allow admins to paste filesystem paths during upload. For libraries this
|
||||
# adds an option to the admin library upload tool allowing admins to paste
|
||||
# filesystem paths to files and directories in a box, and these paths will be
|
||||
# added to a library. Set to True to enable. Please note the security
|
||||
# added to a library. For history uploads, this allows pasting in paths as URIs.
|
||||
# (i.e. prefixed with file://). Set to True to enable. Please note the security
|
||||
# implication that this will give Galaxy Admins access to anything your Galaxy
|
||||
# user has access to.
|
||||
#allow_library_path_paste = False
|
||||
#allow_path_paste = False
|
||||
|
||||
# Users may choose to download multiple files from a library in an archive. By
|
||||
# default, Galaxy allows users to select from a few different archive formats
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
uwsgi:
|
||||
|
||||
# The address and port on which to listen. By default, only listen to
|
||||
# localhost (reports will not be accessible over the network). Use
|
||||
# '0.0.0.0' to listen on all available network interfaces.
|
||||
http: 127.0.0.1:9001
|
||||
|
||||
threads: 8
|
||||
|
||||
http-raw-body: True
|
||||
|
||||
offload-threads: 8
|
||||
|
||||
module: galaxy.webapps.reports.buildapp:uwsgi_app()
|
||||
|
||||
reports:
|
||||
|
||||
# If running behind a proxy server and Galaxy is served from a
|
||||
# subdirectory, enable the proxy-prefix filter and set the prefix in
|
||||
# the [filter:proxy-prefix] section above.
|
||||
#filter-with: proxy-prefix
|
||||
|
||||
# If proxy-prefix is enabled and you're running more than one Galaxy
|
||||
# instance behind one hostname, you will want to set this to the same
|
||||
# path as the prefix in the filter above. This value becomes the
|
||||
# "path" attribute set in the cookie so the cookies from each instance
|
||||
# will not clobber each other.
|
||||
#cookie_path: None
|
||||
|
||||
# Verbosity of console log messages. Acceptable values can be found
|
||||
# here: https://docs.python.org/2/library/logging.html#logging-levels
|
||||
#log_level: DEBUG
|
||||
|
||||
# Database connection Galaxy reports are intended for production
|
||||
# Galaxy instances, so sqlite (and the default value below) is not
|
||||
# supported. An SQLAlchemy connection string should be used specify an
|
||||
# external database.
|
||||
#database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
|
||||
|
||||
# Where dataset files are saved Temporary storage for additional
|
||||
# datasets, this should be shared through the cluster
|
||||
#file_path: database/files
|
||||
|
||||
# Where dataset files are saved Temporary storage for additional
|
||||
# datasets, this should be shared through the cluster
|
||||
#new_file_path: database/tmp
|
||||
|
||||
# Mako templates are compiled as needed and cached for reuse, this
|
||||
# directory is used for the cache
|
||||
#template_cache_path: database/compiled_templates/reports
|
||||
|
||||
# Configuration for debugging middleware
|
||||
#debug: False
|
||||
|
||||
# Check for WSGI compliance.
|
||||
#use_lint: False
|
||||
|
||||
# NEVER enable this on a public site (even test or QA)
|
||||
#use_interactive: True
|
||||
|
||||
# Write thread status periodically to 'heartbeat.log' (careful, uses
|
||||
# disk space rapidly!)
|
||||
#use_heartbeat: True
|
||||
|
||||
# Profiling middleware (cProfile based)
|
||||
#use_profile: True
|
||||
|
||||
# Mail
|
||||
#smtp_server: yourserver@yourfacility.edu
|
||||
|
||||
# Mail
|
||||
#error_email_to: your_bugs@bx.psu.edu
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
uwsgi:
|
||||
|
||||
# The address and port on which to listen. By default, only listen to
|
||||
# localhost (tool_shed will not be accessible over the network). Use
|
||||
# '0.0.0.0' to listen on all available network interfaces.
|
||||
http: 127.0.0.1:9009
|
||||
|
||||
threads: 8
|
||||
|
||||
http-raw-body: True
|
||||
|
||||
offload-threads: 8
|
||||
|
||||
module: galaxy.webapps.tool_shed.buildapp:uwsgi_app()
|
||||
|
||||
tool_shed:
|
||||
|
||||
# Verbosity of console log messages. Acceptable values can be found
|
||||
# here: https://docs.python.org/2/library/logging.html#logging-levels
|
||||
#log_level: DEBUG
|
||||
|
||||
# By default, the Tool Shed uses a SQLite database at
|
||||
# 'database/community.sqlite'. You may use a SQLAlchemy connection
|
||||
# string to specify an external database instead. This string takes
|
||||
# many options which are explained in detail in the config file
|
||||
# documentation.
|
||||
#database_connection: sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE
|
||||
|
||||
# Where the hgweb.config file is stored. The default is the Galaxy
|
||||
# installation directory.
|
||||
#hgweb_config_dir: None
|
||||
|
||||
# Where tool shed repositories are stored.
|
||||
#file_path: database/community_files
|
||||
|
||||
# Temporary storage for additional datasets, this should be shared
|
||||
# through the cluster
|
||||
#new_file_path: database/tmp
|
||||
|
||||
# File containing old-style genome builds
|
||||
#builds_file_path: tool-data/shared/ucsc/builds.txt
|
||||
|
||||
# Format string used when showing date and time information. The
|
||||
# string may contain: - the directives used by Python time.strftime()
|
||||
# function (see
|
||||
# https://docs.python.org/2/library/time.html#time.strftime ), -
|
||||
# $locale (complete format string for the server locale), - $iso8601
|
||||
# (complete format string as specified by ISO 8601 international
|
||||
# standard).
|
||||
#pretty_datetime_format: $locale (UTC)
|
||||
|
||||
# -- Repository and Tool search Using the script located at
|
||||
# scripts/build_ts_whoosh_index.py you can generate search index and
|
||||
# allow full text API searching over the repositories and tools within
|
||||
# the Tool Shed given that you specify the following two config
|
||||
# options.
|
||||
#toolshed_search_on: True
|
||||
|
||||
# -- Repository and Tool search Using the script located at
|
||||
# scripts/build_ts_whoosh_index.py you can generate search index and
|
||||
# allow full text API searching over the repositories and tools within
|
||||
# the Tool Shed given that you specify the following two config
|
||||
# options.
|
||||
#whoosh_index_dir: database/toolshed_whoosh_indexes
|
||||
|
||||
# For searching repositories at /api/repositories:
|
||||
#repo_name_boost: 0.9
|
||||
|
||||
# For searching repositories at /api/repositories:
|
||||
#repo_description_boost: 0.6
|
||||
|
||||
# For searching repositories at /api/repositories:
|
||||
#repo_long_description_boost: 0.5
|
||||
|
||||
# For searching repositories at /api/repositories:
|
||||
#repo_homepage_url_boost: 0.3
|
||||
|
||||
# For searching repositories at /api/repositories:
|
||||
#repo_remote_repository_url_boost: 0.2
|
||||
|
||||
# For searching repositories at /api/repositories:
|
||||
#repo_owner_username_boost: 0.3
|
||||
|
||||
# For searching tools at /api/tools
|
||||
#tool_name_boost: 1.2
|
||||
|
||||
# For searching tools at /api/tools
|
||||
#tool_description_boost: 0.6
|
||||
|
||||
# For searching tools at /api/tools
|
||||
#tool_help_boost: 0.4
|
||||
|
||||
# For searching tools at /api/tools
|
||||
#tool_repo_owner_username: 0.3
|
||||
|
||||
# You can enter tracking code here to track visitor's behavior through
|
||||
# your Google Analytics account. Example: UA-XXXXXXXX-Y
|
||||
#ga_code: None
|
||||
|
||||
# The Tool Shed encodes various internal values when these values will
|
||||
# be output in some format (for example, in a URL or cookie). You
|
||||
# should set a key to be used by the algorithm that encodes and
|
||||
# decodes these values. It can be any string. If left unchanged,
|
||||
# anyone could construct a cookie that would grant them access to
|
||||
# others' sessions. One simple way to generate a value for this is
|
||||
# with the shell command: python -c 'import time; print time.time()'
|
||||
# | md5sum | cut -f 1 -d ' '
|
||||
#id_secret: changethisinproductiontoo
|
||||
|
||||
# User authentication can be delegated to an upstream proxy server
|
||||
# (usually Apache). The upstream proxy should set a REMOTE_USER
|
||||
# header in the request. Enabling remote user disables regular logins.
|
||||
# For more information, see:
|
||||
# https://wiki.galaxyproject.org/Admin/Config/ApacheProxy
|
||||
#use_remote_user: False
|
||||
|
||||
# If use_remote_user is enabled, anyone who can log in to the Galaxy
|
||||
# host may impersonate any other user by simply sending the
|
||||
# appropriate header. Thus a secret shared between the upstream proxy
|
||||
# server, and Galaxy is required. If anyone other than the Galaxy user
|
||||
# is using the server, then apache/nginx should pass a value in the
|
||||
# header 'GX_SECRET' that is identical the one below
|
||||
#remote_user_secret: changethisinproductiontoo
|
||||
|
||||
# Configuration for debugging middleware
|
||||
#debug: False
|
||||
|
||||
# Check for WSGI compliance.
|
||||
#use_lint: False
|
||||
|
||||
# NEVER enable this on a public site (even test or QA)
|
||||
#use_interactive: True
|
||||
|
||||
# Administrative users - set this to a comma-separated list of valid
|
||||
# Tool Shed users (email addresses). These users will have access to
|
||||
# the Admin section of the server, and will have access to create
|
||||
# users, groups, roles, libraries, and more.
|
||||
#admin_users: None
|
||||
|
||||
# Force everyone to log in (disable anonymous access)
|
||||
#require_login: False
|
||||
|
||||
# For use by email messages sent from the tool shed
|
||||
#smtp_server: smtp.your_tool_shed_server
|
||||
|
||||
# For use by email messages sent from the tool shed
|
||||
#email_from: your_tool_shed_email@server
|
||||
|
||||
# If your SMTP server requires a username and password, you can
|
||||
# provide them here (password in cleartext here, but if your server
|
||||
# supports STARTTLS it will be sent over the network encrypted).
|
||||
#smtp_username: None
|
||||
|
||||
# If your SMTP server requires a username and password, you can
|
||||
# provide them here (password in cleartext here, but if your server
|
||||
# supports STARTTLS it will be sent over the network encrypted).
|
||||
#smtp_password: None
|
||||
|
||||
# If your SMTP server requires SSL from the beginning of the
|
||||
# connection
|
||||
#smtp_ssl: False
|
||||
|
||||
# The URL linked by the "Support" link in the "Help" menu.
|
||||
#support_url: https://wiki.galaxyproject.org/Support
|
||||
|
||||
# Address to join mailing list
|
||||
#mailing_join_addr: galaxy-announce-join@bx.psu.edu
|
||||
|
||||
# Write thread status periodically to 'heartbeat.log' (careful, uses
|
||||
# disk space rapidly!)
|
||||
#use_heartbeat: True
|
||||
|
||||
# Profiling middleware (cProfile based)
|
||||
#use_profile: True
|
||||
|
||||
# Enable creation of Galaxy flavor Docker Image
|
||||
#enable_galaxy_flavor_docker_image: False
|
||||
|
||||
# Show a message box under the masthead.
|
||||
#message_box_visible: False
|
||||
|
||||
# Show a message box under the masthead.
|
||||
#message_box_content: None
|
||||
|
||||
# Show a message box under the masthead.
|
||||
#message_box_class: info
|
||||
|
||||
# Serving static files (needed if running standalone)
|
||||
#static_enabled: True
|
||||
|
||||
# Serving static files (needed if running standalone)
|
||||
#static_cache_time: 360
|
||||
|
||||
# Serving static files (needed if running standalone)
|
||||
#static_dir: static/
|
||||
|
||||
# Serving static files (needed if running standalone)
|
||||
#static_images_dir: static/images
|
||||
|
||||
# Serving static files (needed if running standalone)
|
||||
#static_favicon_dir: static/favicon.ico
|
||||
|
||||
# Serving static files (needed if running standalone)
|
||||
#static_scripts_dir: static/scripts/
|
||||
|
||||
# Serving static files (needed if running standalone)
|
||||
#static_style_dir: static/style/blue
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
Galaxy Reports
|
||||
========================================
|
||||
|
||||
Overview
|
||||
----------------------------
|
||||
|
||||
For admins wishing to have more information on the usage of a Galaxy instance,
|
||||
one can configure and use the Galaxy Reports application. This is a separate
|
||||
application that can run beside Galaxy and must target the same database.
|
||||
|
||||
The Galaxy Reports application's easy to use web interface will give you information on the
|
||||
following (and more):
|
||||
|
||||
- Jobs
|
||||
|
||||
- Today's jobs
|
||||
- Jobs per day this month
|
||||
- Jobs in error per day this month
|
||||
- All unfinished jobs
|
||||
- Jobs per month
|
||||
- Jobs in error per month
|
||||
- Jobs per user
|
||||
- Jobs per tool
|
||||
|
||||
- Workflows
|
||||
|
||||
- Workflows per month
|
||||
- Workflows per user
|
||||
|
||||
- Users
|
||||
|
||||
- Registered users
|
||||
- Date of last login
|
||||
- User disk usage
|
||||
|
||||
- System
|
||||
|
||||
- Disk space maintenance
|
||||
|
||||
|
||||
Configuration
|
||||
----------------------------
|
||||
|
||||
- Configure ``config/reports.yml`` in the same manner as your main galaxy instance (i.e., same database connection, but different port). This is a uwsgi YAML configuration file and should contain a reports section with app-specific configuration (options described below).
|
||||
|
||||
- The default port for the reports application is ``9001``, and like Galaxy it only binds to localhost by default.
|
||||
- ``database_connection`` should match the value used in your Galaxy configuration
|
||||
- ``database_connection`` should point at a Postgres database, experimental support for MySQL is available but sqlite is not supported at all.
|
||||
|
||||
- Run reports in a uWSGI server with ``sh run_reports.sh``
|
||||
- Use a web browser and go to the address you configured in ``reports.yml`` (defaults to http://localhost:9001/)
|
||||
|
||||
----------------------------
|
||||
Configuration Options
|
||||
----------------------------
|
||||
|
||||
.. include:: reports_options.rst
|
||||
@@ -0,0 +1,156 @@
|
||||
~~~~~~~~~~~~~~~
|
||||
``filter-with``
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
If running behind a proxy server and Galaxy is served from a
|
||||
subdirectory, enable the proxy-prefix filter and set the prefix in
|
||||
the [filter:proxy-prefix] section above.
|
||||
:Default: proxy-prefix
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~
|
||||
``cookie_path``
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
If proxy-prefix is enabled and you're running more than one Galaxy
|
||||
instance behind one hostname, you will want to set this to the
|
||||
same path as the prefix in the filter above. This value becomes
|
||||
the "path" attribute set in the cookie so the cookies from each
|
||||
instance will not clobber each other.
|
||||
:Default: None
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~
|
||||
``log_level``
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Verbosity of console log messages. Acceptable values can be found
|
||||
here: https://docs.python.org/2/library/logging.html#logging-
|
||||
levels
|
||||
:Default: DEBUG
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
``database_connection``
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Database connection Galaxy reports are intended for production
|
||||
Galaxy instances, so sqlite (and the default value below) is not
|
||||
supported. An SQLAlchemy connection string should be used specify
|
||||
an external database.
|
||||
:Default: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~
|
||||
``file_path``
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Where dataset files are saved Temporary storage for additional
|
||||
datasets, this should be shared through the cluster
|
||||
:Default: database/files
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~
|
||||
``new_file_path``
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Where dataset files are saved Temporary storage for additional
|
||||
datasets, this should be shared through the cluster
|
||||
:Default: database/tmp
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
``template_cache_path``
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Mako templates are compiled as needed and cached for reuse, this
|
||||
directory is used for the cache
|
||||
:Default: database/compiled_templates/reports
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~
|
||||
``debug``
|
||||
~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Configuration for debugging middleware
|
||||
:Default: False
|
||||
:Type: bool
|
||||
|
||||
|
||||
~~~~~~~~~~~~
|
||||
``use_lint``
|
||||
~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Check for WSGI compliance.
|
||||
:Default: False
|
||||
:Type: bool
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
``use_interactive``
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
NEVER enable this on a public site (even test or QA)
|
||||
:Default: True
|
||||
:Type: bool
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~
|
||||
``use_heartbeat``
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Write thread status periodically to 'heartbeat.log' (careful, uses
|
||||
disk space rapidly!)
|
||||
:Default: True
|
||||
:Type: bool
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~
|
||||
``use_profile``
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Profiling middleware (cProfile based)
|
||||
:Default: True
|
||||
:Type: bool
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~
|
||||
``smtp_server``
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Mail
|
||||
:Default: yourserver@yourfacility.edu
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
``error_email_to``
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Mail
|
||||
:Default: your_bugs@bx.psu.edu
|
||||
:Type: str
|
||||
|
||||
|
||||
|
||||
@@ -393,8 +393,12 @@ class Configuration( object ):
|
||||
self.ftp_upload_dir_template = kwargs.get( 'ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep )
|
||||
self.ftp_upload_purge = string_as_bool( kwargs.get( 'ftp_upload_purge', 'True' ) )
|
||||
self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
|
||||
self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
|
||||
self.allow_path_paste = string_as_bool( kwargs.get('allow_path_paste', False ) )
|
||||
# Support older library-specific path paste option but just default to the new
|
||||
# allow_path_paste value.
|
||||
self.allow_library_path_paste = string_as_bool( kwargs.get( 'allow_library_path_paste', self.allow_path_paste ) )
|
||||
self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )
|
||||
self.check_upload_content = string_as_bool( kwargs.get( 'check_upload_content', True ) )
|
||||
self.watch_tools = kwargs.get( 'watch_tools', 'false' )
|
||||
self.watch_tool_data_dir = kwargs.get( 'watch_tool_data_dir', 'false' )
|
||||
# On can mildly speed up Galaxy startup time by disabling index of help,
|
||||
|
||||
@@ -1174,6 +1174,40 @@ class Xlsx(Binary):
|
||||
Binary.register_sniffable_binary_format("xlsx", "xlsx", Xlsx)
|
||||
|
||||
|
||||
class ExcelXls( Binary ):
|
||||
"""Class describing an Excel (xls) file"""
|
||||
file_ext = "excel.xls"
|
||||
edam_format = "format_3468"
|
||||
|
||||
def sniff( self, filename ):
|
||||
mime_type = subprocess.check_output("file --mime-type '{}'".format(filename), shell=True).rstrip()
|
||||
if mime_type.find("application/vnd.ms-excel") != -1:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def get_mime( self ):
|
||||
"""Returns the mime type of the datatype"""
|
||||
return 'application/vnd.ms-excel'
|
||||
|
||||
def set_peek( self, dataset, is_multi_byte=False ):
|
||||
if not dataset.dataset.purged:
|
||||
dataset.peek = "Microsoft Excel XLS file"
|
||||
dataset.blurb = data.nice_size( dataset.get_size() )
|
||||
else:
|
||||
dataset.peek = 'file does not exist'
|
||||
dataset.blurb = 'file purged from disk'
|
||||
|
||||
def display_peek( self, dataset ):
|
||||
try:
|
||||
return dataset.peek
|
||||
except:
|
||||
return "Microsoft Excel XLS file (%s)" % ( data.nice_size( dataset.get_size() ) )
|
||||
|
||||
|
||||
Binary.register_sniffable_binary_format("excel.xls", "excel.xls", ExcelXls)
|
||||
|
||||
|
||||
class Sra( Binary ):
|
||||
""" Sequence Read Archive (SRA) datatype originally from mdshw5/sra-tools-galaxy"""
|
||||
file_ext = 'sra'
|
||||
|
||||
@@ -354,7 +354,7 @@ class FPS(GenericMolFile):
|
||||
|
||||
class OBFS(Binary):
|
||||
"""OpenBabel Fastsearch format (fs)."""
|
||||
file_ext = 'fs'
|
||||
file_ext = 'obfs'
|
||||
composite_type = 'basic'
|
||||
allow_datatype_change = False
|
||||
|
||||
|
||||
@@ -383,6 +383,9 @@ def guess_ext( fname, sniff_order, is_multi_byte=False ):
|
||||
>>> fname = get_test_fname('diamond_db.dmnd')
|
||||
>>> guess_ext(fname, sniff_order)
|
||||
'dmnd'
|
||||
>>> fname = get_test_fname('1.xls')
|
||||
>>> guess_ext(fname, sniff_order)
|
||||
'excel.xls'
|
||||
"""
|
||||
file_ext = None
|
||||
for datatype in sniff_order:
|
||||
|
||||
Binary file not shown.
@@ -22,7 +22,7 @@ class ConditionalDependencies( object ):
|
||||
self.get_conditional_requirements()
|
||||
|
||||
def parse_configs( self ):
|
||||
self.config = load_app_properties( ini_file=self.config_file )
|
||||
self.config = load_app_properties( config_file=self.config_file )
|
||||
job_conf_xml = self.config.get(
|
||||
"job_config_file",
|
||||
join( dirname( self.config_file ), 'job_conf.xml' ) )
|
||||
|
||||
@@ -6,6 +6,7 @@ SQLAlchemy==1.0.15
|
||||
mercurial==3.7.3
|
||||
numpy==1.9.2
|
||||
pycrypto==2.6.1
|
||||
uWSGI==2.0.15
|
||||
|
||||
# Install python_lzo if you want to support indexed access to lzo-compressed
|
||||
# locally cached maf files via bx-python
|
||||
@@ -32,6 +33,11 @@ Whoosh==2.7.4
|
||||
testfixtures==4.10.0
|
||||
galaxy_sequence_utils==1.0.2
|
||||
|
||||
# pykwalify and dependencies
|
||||
pykwalify==1.5.1
|
||||
python-dateutil==2.5.3
|
||||
docopt==0.6.2
|
||||
|
||||
# Cheetah and dependencies
|
||||
Cheetah==2.4.4
|
||||
Markdown==2.6.3
|
||||
|
||||
@@ -293,6 +293,9 @@ class WorkflowContentsManager(UsesAnnotations):
|
||||
# the local Galaxy instance. Each tuple in the list of missing_tool_tups
|
||||
# will be ( tool_id, tool_name, tool_version ).
|
||||
missing_tool_tups = []
|
||||
for step_dict in self.__walk_step_dicts( data ):
|
||||
self.__load_subworkflows( trans, step_dict )
|
||||
|
||||
for step_dict in self.__walk_step_dicts( data ):
|
||||
module, step = self.__module_from_dict( trans, steps, steps_by_external_id, step_dict, **kwds )
|
||||
is_tool = is_tool_module_type( module.type )
|
||||
@@ -815,6 +818,14 @@ class WorkflowContentsManager(UsesAnnotations):
|
||||
|
||||
yield step_dict
|
||||
|
||||
def __load_subworkflows( self, trans, step_dict ):
|
||||
step_type = step_dict.get("type", None)
|
||||
if step_type == "subworkflow":
|
||||
subworkflow = self.__load_subworkflow_from_step_dict(
|
||||
trans, step_dict
|
||||
)
|
||||
step_dict["subworkflow"] = subworkflow
|
||||
|
||||
def __module_from_dict( self, trans, steps, steps_by_external_id, step_dict, **kwds ):
|
||||
""" Create a WorkflowStep model object and corresponding module
|
||||
representing type-specific functionality from the incoming dictionary.
|
||||
@@ -826,12 +837,6 @@ class WorkflowContentsManager(UsesAnnotations):
|
||||
step.uuid = step_dict["uuid"]
|
||||
if "label" in step_dict:
|
||||
step.label = step_dict["label"]
|
||||
step_type = step_dict.get("type", None)
|
||||
if step_type == "subworkflow":
|
||||
subworkflow = self.__load_subworkflow_from_step_dict(
|
||||
trans, step_dict
|
||||
)
|
||||
step_dict["subworkflow"] = subworkflow
|
||||
|
||||
module = module_factory.from_dict( trans, step_dict, **kwds )
|
||||
self.__set_default_label( step, module, step_dict.get( 'tool_state' ) )
|
||||
|
||||
@@ -3,6 +3,7 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...).
|
||||
"""
|
||||
import logging
|
||||
|
||||
from galaxy.util import listify
|
||||
from galaxy.util.properties import find_config_file, load_app_properties
|
||||
|
||||
|
||||
@@ -16,22 +17,23 @@ DATABASE = {
|
||||
"galaxy":
|
||||
{
|
||||
'repo': 'lib/galaxy/model/migrate',
|
||||
'old_config_file': 'universe_wsgi.ini',
|
||||
'old_config_files': ['universe_wsgi.ini'],
|
||||
'default_sqlite_file': './database/universe.sqlite',
|
||||
'config_override': 'GALAXY_CONFIG_',
|
||||
},
|
||||
"tool_shed":
|
||||
{
|
||||
'repo': 'lib/galaxy/webapps/tool_shed/model/migrate',
|
||||
'config_file': 'config/tool_shed.ini',
|
||||
'old_config_file': 'tool_shed_wsgi.ini',
|
||||
'config_file': 'config/tool_shed.yml',
|
||||
'old_config_files': ['config/tool_shed.ini', 'tool_shed_wsgi.ini'],
|
||||
'default_sqlite_file': './database/community.sqlite',
|
||||
'config_override': 'TOOL_SHED_CONFIG_',
|
||||
'config_section': 'tool_shed',
|
||||
},
|
||||
"install":
|
||||
{
|
||||
'repo': 'lib/galaxy/model/tool_shed_install/migrate',
|
||||
'old_config_file': 'universe_wsgi.ini',
|
||||
'old_config_files': ['universe_wsgi.ini'],
|
||||
'config_prefix': 'install_',
|
||||
'default_sqlite_file': './database/install.sqlite',
|
||||
'config_override': 'GALAXY_INSTALL_CONFIG_',
|
||||
@@ -39,14 +41,14 @@ DATABASE = {
|
||||
}
|
||||
|
||||
|
||||
def read_config_file_arg( argv, default, old_default, cwd=None ):
|
||||
def read_config_file_arg( argv, default, old_defaults, cwd=None ):
|
||||
config_file = None
|
||||
if '-c' in argv:
|
||||
pos = argv.index( '-c' )
|
||||
argv.pop(pos)
|
||||
config_file = argv.pop( pos )
|
||||
|
||||
return find_config_file( default, old_default, config_file, cwd=cwd )
|
||||
old_defaults = listify(old_defaults)
|
||||
return find_config_file( default, old_defaults, config_file, cwd=cwd )
|
||||
|
||||
|
||||
def get_config( argv, cwd=None ):
|
||||
@@ -83,14 +85,18 @@ def get_config( argv, cwd=None ):
|
||||
database_defaults = DATABASE[ database ]
|
||||
|
||||
default = database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE )
|
||||
old_default = database_defaults.get( 'old_config_file' )
|
||||
config_file = read_config_file_arg( argv, default, old_default, cwd=cwd )
|
||||
old_defaults = database_defaults.get( 'old_config_files' )
|
||||
config_file = read_config_file_arg( argv, default, old_defaults, cwd=cwd )
|
||||
repo = database_defaults[ 'repo' ]
|
||||
config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX )
|
||||
config_override = database_defaults.get( 'config_override', 'GALAXY_CONFIG_' )
|
||||
default_sqlite_file = database_defaults[ 'default_sqlite_file' ]
|
||||
|
||||
properties = load_app_properties( ini_file=config_file, config_prefix=config_override )
|
||||
if config_file.endswith(".yml") or config_file.endswith(".yml.sample"):
|
||||
config_section = database_defaults.get( 'config_section', None )
|
||||
else:
|
||||
# An .ini file - just let load_app_properties find app:main.
|
||||
config_section = None
|
||||
properties = load_app_properties( config_file=config_file, config_prefix=config_override, config_section=config_section )
|
||||
|
||||
if ("%sdatabase_connection" % config_prefix) in properties:
|
||||
db_url = properties[ "%sdatabase_connection" % config_prefix ]
|
||||
|
||||
@@ -347,9 +347,11 @@ def create_paramfile( trans, uploaded_datasets ):
|
||||
link_data_only=link_data_only,
|
||||
uuid=uuid_str,
|
||||
to_posix_lines=getattr(uploaded_dataset, "to_posix_lines", True),
|
||||
auto_decompress=getattr(uploaded_dataset, "auto_decompress", True),
|
||||
purge_source=purge_source,
|
||||
space_to_tab=uploaded_dataset.space_to_tab,
|
||||
in_place=trans.app.config.external_chown_script is None,
|
||||
check_content=trans.app.config.check_upload_content,
|
||||
path=uploaded_dataset.path )
|
||||
# TODO: This will have to change when we start bundling inputs.
|
||||
# Also, in_place above causes the file to be left behind since the
|
||||
|
||||
@@ -11,6 +11,10 @@ from six import (
|
||||
)
|
||||
|
||||
from galaxy.datatypes import sniff
|
||||
from galaxy.exceptions import (
|
||||
AdminRequiredException,
|
||||
ConfigDoesNotAllowException,
|
||||
)
|
||||
from galaxy.util import (
|
||||
inflector,
|
||||
relpath,
|
||||
@@ -289,7 +293,6 @@ class UploadDataset( Group ):
|
||||
def get_uploaded_datasets( self, trans, context, override_name=None, override_info=None ):
|
||||
def get_data_file_filename( data_file, override_name=None, override_info=None, purge=True ):
|
||||
dataset_name = override_name
|
||||
dataset_info = override_info
|
||||
|
||||
def get_file_name( file_name ):
|
||||
file_name = file_name.split( '\\' )[-1]
|
||||
@@ -299,8 +302,6 @@ class UploadDataset( Group ):
|
||||
# Use the existing file
|
||||
if not dataset_name and 'filename' in data_file:
|
||||
dataset_name = get_file_name( data_file['filename'] )
|
||||
if not dataset_info:
|
||||
dataset_info = 'uploaded file'
|
||||
return Bunch( type='file', path=data_file['local_filename'], name=dataset_name, purge_source=purge )
|
||||
except:
|
||||
# The uploaded file should've been persisted by the upload tool action
|
||||
@@ -310,26 +311,42 @@ class UploadDataset( Group ):
|
||||
url_paste_file = group_incoming.get( 'url_paste', None )
|
||||
if url_paste_file is not None:
|
||||
url_paste = open( url_paste_file, 'r' ).read( 1024 )
|
||||
if url_paste.lstrip().lower().startswith( 'http://' ) or url_paste.lstrip().lower().startswith( 'ftp://' ) or url_paste.lstrip().lower().startswith( 'https://' ):
|
||||
|
||||
def start_of_url(content):
|
||||
start_of_url_paste = content.lstrip()[0:8].lower()
|
||||
looks_like_url = False
|
||||
for url_prefix in ["http://", "https://", "ftp://", "file://"]:
|
||||
if start_of_url_paste.startswith(url_prefix):
|
||||
looks_like_url = True
|
||||
break
|
||||
|
||||
return looks_like_url
|
||||
|
||||
if start_of_url(url_paste):
|
||||
url_paste = url_paste.replace( '\r', '' ).split( '\n' )
|
||||
for line in url_paste:
|
||||
line = line.strip()
|
||||
if line:
|
||||
if not line.lower().startswith( 'http://' ) and not line.lower().startswith( 'ftp://' ) and not line.lower().startswith( 'https://' ):
|
||||
if not start_of_url(line):
|
||||
continue # non-url line, ignore
|
||||
dataset_name = override_name
|
||||
if not dataset_name:
|
||||
|
||||
if "file://" in line:
|
||||
if not trans.user_is_admin():
|
||||
raise AdminRequiredException()
|
||||
elif not trans.app.config.allow_path_paste:
|
||||
raise ConfigDoesNotAllowException()
|
||||
upload_path = line[len("file://"):]
|
||||
dataset_name = os.path.basename(upload_path)
|
||||
else:
|
||||
dataset_name = line
|
||||
dataset_info = override_info
|
||||
if not dataset_info:
|
||||
dataset_info = 'uploaded url'
|
||||
|
||||
if override_name:
|
||||
dataset_name = override_name
|
||||
yield Bunch( type='url', path=line, name=dataset_name )
|
||||
else:
|
||||
dataset_name = dataset_info = 'Pasted Entry' # we need to differentiate between various url pastes here
|
||||
dataset_name = 'Pasted Entry' # we need to differentiate between various url pastes here
|
||||
if override_name:
|
||||
dataset_name = override_name
|
||||
if override_info:
|
||||
dataset_info = override_info
|
||||
yield Bunch( type='file', path=url_paste_file, name=dataset_name )
|
||||
|
||||
def get_one_filename( context ):
|
||||
@@ -343,6 +360,9 @@ class UploadDataset( Group ):
|
||||
to_posix_lines = False
|
||||
if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]:
|
||||
to_posix_lines = True
|
||||
auto_decompress = False
|
||||
if context.get( 'auto_decompress', None ) not in [ "None", None, False ]:
|
||||
auto_decompress = True
|
||||
space_to_tab = False
|
||||
if context.get( 'space_to_tab', None ) not in [ "None", None, False ]:
|
||||
space_to_tab = True
|
||||
@@ -382,6 +402,7 @@ class UploadDataset( Group ):
|
||||
if file_bunch.path:
|
||||
break
|
||||
file_bunch.to_posix_lines = to_posix_lines
|
||||
file_bunch.auto_decompress = auto_decompress
|
||||
file_bunch.space_to_tab = space_to_tab
|
||||
file_bunch.uuid = uuid
|
||||
return file_bunch, warnings
|
||||
@@ -396,6 +417,9 @@ class UploadDataset( Group ):
|
||||
to_posix_lines = False
|
||||
if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]:
|
||||
to_posix_lines = True
|
||||
auto_decompress = False
|
||||
if context.get( 'auto_decompress', None ) not in [ "None", None, False ]:
|
||||
auto_decompress = True
|
||||
space_to_tab = False
|
||||
if context.get( 'space_to_tab', None ) not in [ "None", None, False ]:
|
||||
space_to_tab = True
|
||||
@@ -403,12 +427,14 @@ class UploadDataset( Group ):
|
||||
file_bunch.uuid = uuid
|
||||
if file_bunch.path:
|
||||
file_bunch.to_posix_lines = to_posix_lines
|
||||
file_bunch.auto_decompress = auto_decompress
|
||||
file_bunch.space_to_tab = space_to_tab
|
||||
rval.append( file_bunch )
|
||||
for file_bunch in get_url_paste_urls_or_filename( context, override_name=name, override_info=info ):
|
||||
if file_bunch.path:
|
||||
file_bunch.uuid = uuid
|
||||
file_bunch.to_posix_lines = to_posix_lines
|
||||
file_bunch.auto_decompress = auto_decompress
|
||||
file_bunch.space_to_tab = space_to_tab
|
||||
rval.append( file_bunch )
|
||||
# look for files uploaded via FTP
|
||||
@@ -446,6 +472,7 @@ class UploadDataset( Group ):
|
||||
file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info, purge=purge )
|
||||
if file_bunch.path:
|
||||
file_bunch.to_posix_lines = to_posix_lines
|
||||
file_bunch.auto_decompress = auto_decompress
|
||||
file_bunch.space_to_tab = space_to_tab
|
||||
rval.append( file_bunch )
|
||||
return rval
|
||||
@@ -488,12 +515,14 @@ class UploadDataset( Group ):
|
||||
temp_name, is_multi_byte = sniff.stream_to_file( StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' )
|
||||
dataset.primary_file = temp_name
|
||||
dataset.to_posix_lines = True
|
||||
dataset.auto_decompress = True
|
||||
dataset.space_to_tab = False
|
||||
else:
|
||||
file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] )
|
||||
writable_files_offset = 1
|
||||
dataset.primary_file = file_bunch.path
|
||||
dataset.to_posix_lines = file_bunch.to_posix_lines
|
||||
dataset.auto_decompress = file_bunch.auto_decompress
|
||||
dataset.space_to_tab = file_bunch.space_to_tab
|
||||
dataset.warnings.extend( warnings )
|
||||
if dataset.primary_file is None: # remove this before finish, this should create an empty dataset
|
||||
|
||||
@@ -14,6 +14,8 @@ HTML_CHECK_LINES = 100
|
||||
def check_html( file_path, chunk=None ):
|
||||
if chunk is None:
|
||||
temp = open( file_path, "U" )
|
||||
elif hasattr(chunk, "splitlines"):
|
||||
temp = chunk.splitlines()
|
||||
else:
|
||||
temp = chunk
|
||||
regexp1 = re.compile( "<A\s+[^>]*HREF[^>]+>", re.I )
|
||||
@@ -55,7 +57,7 @@ def check_binary( name, file_path=True ):
|
||||
return is_binary
|
||||
|
||||
|
||||
def check_gzip( file_path ):
|
||||
def check_gzip( file_path, check_content=True ):
|
||||
# This method returns a tuple of booleans representing ( is_gzipped, is_valid )
|
||||
# Make sure we have a gzipped file
|
||||
try:
|
||||
@@ -75,6 +77,10 @@ def check_gzip( file_path ):
|
||||
return ( True, True )
|
||||
except:
|
||||
return( False, False )
|
||||
|
||||
if not check_content:
|
||||
return ( True, True )
|
||||
|
||||
CHUNK_SIZE = 2 ** 15 # 32Kb
|
||||
gzipped_file = gzip.GzipFile( file_path, mode='rb' )
|
||||
chunk = gzipped_file.read( CHUNK_SIZE )
|
||||
@@ -85,7 +91,7 @@ def check_gzip( file_path ):
|
||||
return ( True, True )
|
||||
|
||||
|
||||
def check_bz2( file_path ):
|
||||
def check_bz2( file_path, check_content=True ):
|
||||
try:
|
||||
temp = open( file_path, "U" )
|
||||
magic_check = temp.read( 3 )
|
||||
@@ -94,6 +100,10 @@ def check_bz2( file_path ):
|
||||
return ( False, False )
|
||||
except:
|
||||
return( False, False )
|
||||
|
||||
if not check_content:
|
||||
return ( True, True )
|
||||
|
||||
CHUNK_SIZE = 2 ** 15 # reKb
|
||||
bzipped_file = bz2.BZ2File( file_path, mode='rb' )
|
||||
chunk = bzipped_file.read( CHUNK_SIZE )
|
||||
@@ -111,12 +121,12 @@ def check_zip( file_path ):
|
||||
|
||||
|
||||
def is_bz2( file_path ):
|
||||
is_bz2, is_valid = check_bz2( file_path )
|
||||
is_bz2, is_valid = check_bz2( file_path, check_content=False )
|
||||
return is_bz2
|
||||
|
||||
|
||||
def is_gzip( file_path ):
|
||||
is_gzipped, is_valid = check_gzip( file_path )
|
||||
is_gzipped, is_valid = check_gzip( file_path, check_content=False )
|
||||
return is_gzipped
|
||||
|
||||
|
||||
|
||||
@@ -6,14 +6,20 @@ import os
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
from six import iteritems
|
||||
from six.moves.configparser import ConfigParser
|
||||
|
||||
from galaxy.util import listify
|
||||
|
||||
def find_config_file(default, old_default, explicit, cwd=None):
|
||||
|
||||
def find_config_file(default, old_defaults, explicit, cwd=None):
|
||||
old_defaults = listify(old_defaults)
|
||||
if cwd is not None:
|
||||
default = os.path.join(cwd, default)
|
||||
old_default = os.path.join(cwd, old_default)
|
||||
for i in range(len(old_defaults)):
|
||||
old_defaults[i] = os.path.join(cwd, old_defaults[i])
|
||||
if explicit is not None:
|
||||
explicit = os.path.join(cwd, explicit)
|
||||
|
||||
@@ -23,33 +29,46 @@ def find_config_file(default, old_default, explicit, cwd=None):
|
||||
else:
|
||||
raise Exception("Problem determining Galaxy's configuration - the specified configuration file cannot be found.")
|
||||
else:
|
||||
if not os.path.exists( default ) and os.path.exists( old_default ):
|
||||
config_file = old_default
|
||||
elif os.path.exists( default ):
|
||||
config_file = None
|
||||
if os.path.exists(default):
|
||||
config_file = default
|
||||
else:
|
||||
if config_file is None:
|
||||
for old_default in old_defaults:
|
||||
if os.path.exists(old_default):
|
||||
config_file = old_default
|
||||
|
||||
if config_file is None:
|
||||
config_file = default + ".sample"
|
||||
|
||||
return config_file
|
||||
|
||||
|
||||
def load_app_properties(
|
||||
kwds={},
|
||||
ini_file=None,
|
||||
ini_section="app:main",
|
||||
ini_section=None,
|
||||
config_file=None,
|
||||
config_section=None,
|
||||
config_prefix="GALAXY_CONFIG_"
|
||||
):
|
||||
properties = kwds.copy() if kwds else {}
|
||||
if ini_file:
|
||||
defaults = {
|
||||
'here': os.path.dirname(os.path.abspath(ini_file)),
|
||||
'__file__': os.path.abspath(ini_file)
|
||||
}
|
||||
parser = NicerConfigParser(ini_file, defaults=defaults)
|
||||
parser.optionxform = str # Don't lower-case keys
|
||||
with open(ini_file) as f:
|
||||
parser.read_file(f)
|
||||
if config_file is None:
|
||||
config_file = ini_file
|
||||
config_section = ini_section
|
||||
|
||||
properties.update( dict( parser.items( ini_section ) ) )
|
||||
if config_file:
|
||||
if not config_file.endswith(".yml") and not config_file.endswith(".yml.sample"):
|
||||
if config_section is None:
|
||||
config_section = "app:main"
|
||||
parser = nice_config_parser(config_file)
|
||||
properties.update( dict( parser.items( config_section ) ) )
|
||||
else:
|
||||
if config_section is None:
|
||||
config_section = "galaxy"
|
||||
|
||||
with open(config_file, "r") as f:
|
||||
raw_properties = yaml.load(f)
|
||||
properties = raw_properties[config_section] or {}
|
||||
|
||||
override_prefix = "%sOVERRIDE_" % config_prefix
|
||||
for key in os.environ:
|
||||
@@ -64,6 +83,18 @@ def load_app_properties(
|
||||
return properties
|
||||
|
||||
|
||||
def nice_config_parser(path):
|
||||
defaults = {
|
||||
'here': os.path.dirname(os.path.abspath(path)),
|
||||
'__file__': os.path.abspath(path)
|
||||
}
|
||||
parser = NicerConfigParser(path, defaults=defaults)
|
||||
parser.optionxform = str # Don't lower-case keys
|
||||
with open(path) as f:
|
||||
parser.read_file(f)
|
||||
return parser
|
||||
|
||||
|
||||
class NicerConfigParser(ConfigParser):
|
||||
|
||||
def __init__(self, filename, *args, **kw):
|
||||
|
||||
@@ -954,6 +954,24 @@ class GalaxyWebTransaction( base.DefaultWebTransaction,
|
||||
return str(template)
|
||||
|
||||
|
||||
def build_native_uwsgi_app( paste_factory, config_section ):
|
||||
"""uwsgi can load paste factories with --ini-paste, but this builds non-paste uwsgi apps.
|
||||
|
||||
In particular these are useful with --yaml or --json for config."""
|
||||
import uwsgi
|
||||
uwsgi_opt = uwsgi.opt
|
||||
config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
|
||||
if not config_file:
|
||||
# Probably loaded via --ini-paste - expect paste app.
|
||||
return None
|
||||
|
||||
uwsgi_app = paste_factory(uwsgi.opt, load_app_kwds={
|
||||
"config_file": config_file,
|
||||
"config_section": config_section,
|
||||
})
|
||||
return uwsgi_app
|
||||
|
||||
|
||||
def build_url_map( app, global_conf, local_conf ):
|
||||
from paste.urlmap import URLMap
|
||||
from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static
|
||||
|
||||
@@ -0,0 +1,710 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
from collections import namedtuple
|
||||
from collections import OrderedDict
|
||||
import copy
|
||||
import os
|
||||
import shutil
|
||||
import string
|
||||
import sys
|
||||
import tempfile
|
||||
from textwrap import TextWrapper
|
||||
import urllib2
|
||||
|
||||
import six
|
||||
from six import StringIO
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
from pykwalify.core import Core
|
||||
except ImportError:
|
||||
Core = None
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)))
|
||||
|
||||
from galaxy.util.properties import nice_config_parser
|
||||
from galaxy.util import safe_makedirs
|
||||
|
||||
DESCRIPTION = "Convert configuration files."
|
||||
|
||||
APP_DESCRIPTION = """Application to target for operation (i.e. galaxy, tool_shed, or reports))"""
|
||||
DRY_RUN_DESCRIPTION = """If this action modifies files, just print what would be the result and continue."""
|
||||
UNKNOWN_OPTION_MESSAGE = "Option [%s] not found in schema - either it is invalid or the Galaxy team hasn't documented it. If invalid, you should manually remove it. If the option is valid but undocumented, please file an issue with the Galaxy team."
|
||||
USING_SAMPLE_MESSAGE = "Path [%s] not a file, using sample."
|
||||
EXTRA_SERVER_MESSAGE = "Additional server section after [%s] encountered [%s], will be ignored."
|
||||
MISSING_FILTER_TYPE_MESSAGE = "Missing filter type for section [%s], it will be ignored."
|
||||
UNHANDLED_FILTER_TYPE_MESSAGE = "Unhandled filter type encountered [%s] for section [%s]."
|
||||
NO_APP_MAIN_MESSAGE = "No app:main section found, using application defaults throughout."
|
||||
YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ")
|
||||
RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent=" ", subsequent_indent=" ")
|
||||
UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml"
|
||||
|
||||
App = namedtuple(
|
||||
"App",
|
||||
["config_paths", "default_port", "expected_app_factories", "destination", "schema_path", "uwsgi_module"]
|
||||
)
|
||||
|
||||
UWSGI_OPTIONS = OrderedDict([
|
||||
('http', {
|
||||
'desc': """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use '0.0.0.0' to listen on all available network interfaces.""",
|
||||
'default': '127.0.0.1:$default_port',
|
||||
'type': 'str',
|
||||
}),
|
||||
('threads', {
|
||||
'default': 8,
|
||||
'type': 'int',
|
||||
}),
|
||||
# ('route-uri', {
|
||||
# 'default': '^/proxy/ goto:proxy'
|
||||
# }),
|
||||
# ('route', {
|
||||
# 'default': '.* last:'
|
||||
# }),
|
||||
# ('route-label', {
|
||||
# 'default': 'proxy'
|
||||
# }),
|
||||
# ('route-run', {
|
||||
# 'default': 'rpcvar:TARGET_HOST galaxy_dynamic_proxy_mapper ${HTTP_HOST} ${cookie[galaxysession]}'
|
||||
# }),
|
||||
# ('route-run', {
|
||||
# 'default': "['log:Proxy ${HTTP_HOST} to ${TARGET_HOST}', 'httpdumb:${TARGET_HOST}']",
|
||||
# }),
|
||||
('http-raw-body', {
|
||||
'default': 'True'
|
||||
}),
|
||||
('offload-threads', {
|
||||
'default': '8',
|
||||
}),
|
||||
('module', {
|
||||
'default': '$uwsgi_module',
|
||||
'type': 'str',
|
||||
})
|
||||
])
|
||||
|
||||
DROP_OPTION_VALUE = object()
|
||||
|
||||
|
||||
class _OptionAction(object):
|
||||
|
||||
def converted(self, args, app_desc, key, value):
|
||||
pass
|
||||
|
||||
def lint(self, args, app_desc, key, value):
|
||||
pass
|
||||
|
||||
|
||||
class _DeprecatedAndDroppedAction(_OptionAction):
|
||||
|
||||
def converted(self, args, app_desc, key, value):
|
||||
print("Option [%s] has been deprecated and dropped. It is not included in converted configuration." % key)
|
||||
return DROP_OPTION_VALUE
|
||||
|
||||
def lint(self, args, app_desc, key, value):
|
||||
print("Option [%s] has been deprecated. Option should be dropped without replacement." % key)
|
||||
|
||||
|
||||
class _PasteAppFactoryAction(_OptionAction):
|
||||
|
||||
def converted(self, args, app_desc, key, value):
|
||||
if value not in app_desc.expected_app_factories:
|
||||
raise Exception("Ending convert process - unknown paste factory encountered [%s]" % value)
|
||||
return DROP_OPTION_VALUE
|
||||
|
||||
def lint(self, args, app_desc, key, value):
|
||||
if value not in app_desc.expected_app_factories:
|
||||
print("Problem - unknown paste app factory encountered [%s]" % value)
|
||||
|
||||
|
||||
class _ProductionNotReady(_OptionAction):
|
||||
|
||||
def __init__(self, unsafe_value):
|
||||
self.unsafe_value = unsafe_value
|
||||
|
||||
def lint(self, args, app_desc, key, value):
|
||||
if str(value).lower() == str(self.unsafe_value).lower():
|
||||
template = "Problem - option [%s] should not be set to [%s] in production environments - it is unsafe."
|
||||
message = template % (key, value)
|
||||
print(message)
|
||||
|
||||
|
||||
class _HandleFilterWithAction(_OptionAction):
|
||||
|
||||
def converted(self, args, app_desc, key, value):
|
||||
print("filter-with converted to prefixed module load of uwsgi module, dropping from converted configuration")
|
||||
return DROP_OPTION_VALUE
|
||||
|
||||
|
||||
OPTION_ACTIONS = {
|
||||
'use_beaker_session': _DeprecatedAndDroppedAction(),
|
||||
'session_type': _DeprecatedAndDroppedAction(),
|
||||
'session_data_dir': _DeprecatedAndDroppedAction(),
|
||||
'session_key': _DeprecatedAndDroppedAction(),
|
||||
'session_secret': _DeprecatedAndDroppedAction(),
|
||||
'paste.app_factory': _PasteAppFactoryAction(),
|
||||
'filter-with': _HandleFilterWithAction(),
|
||||
'debug': _ProductionNotReady(True),
|
||||
'serve_xss_vulnerable_mimetypes': _ProductionNotReady(True),
|
||||
'use_printdebug': _ProductionNotReady(True),
|
||||
'use_interactive': _ProductionNotReady(True),
|
||||
'id_secret': _ProductionNotReady('USING THE DEFAULT IS NOT SECURE!'),
|
||||
'master_api_key': _ProductionNotReady('changethis'),
|
||||
}
|
||||
|
||||
|
||||
def _app_name(self):
|
||||
return os.path.splitext(os.path.basename(self.destination))[0]
|
||||
|
||||
|
||||
def _sample_destination(self):
|
||||
return self.destination + ".sample"
|
||||
|
||||
|
||||
def _schema(self):
|
||||
return AppSchema(self)
|
||||
|
||||
|
||||
App.app_name = property(_app_name)
|
||||
App.sample_destination = property(_sample_destination)
|
||||
App.schema = property(_schema)
|
||||
|
||||
OptionValue = namedtuple("OptionValue", ["name", "value", "option"] )
|
||||
|
||||
|
||||
UNKNOWN_OPTION = {
|
||||
"type": "str",
|
||||
"required": False,
|
||||
"unknown_option": True,
|
||||
"desc": "Unknown option, may want to remove or report to Galaxy team."
|
||||
}
|
||||
|
||||
OPTION_DEFAULTS = {
|
||||
"type": "str",
|
||||
"unknown_option": False,
|
||||
"default": None,
|
||||
"desc": None,
|
||||
}
|
||||
|
||||
|
||||
class Schema(object):
|
||||
|
||||
def __init__(self, mapping):
|
||||
self.app_schema = mapping
|
||||
|
||||
def get_app_option(self, name):
|
||||
try:
|
||||
raw_option = self.app_schema[name]
|
||||
except KeyError:
|
||||
raw_option = UNKNOWN_OPTION
|
||||
option = OPTION_DEFAULTS.copy()
|
||||
option.update(raw_option)
|
||||
return option
|
||||
|
||||
|
||||
class AppSchema(Schema):
|
||||
|
||||
def __init__(self, app_desc):
|
||||
schema_path = app_desc.schema_path
|
||||
app_name = app_desc.app_name
|
||||
with open(schema_path, "r") as f:
|
||||
config_all = _ordered_load(f)
|
||||
self.raw_schema = config_all
|
||||
app_schema = config_all["mapping"][app_name]
|
||||
super(AppSchema, self).__init__(app_schema["mapping"])
|
||||
|
||||
def get_app_option(self, name):
|
||||
try:
|
||||
raw_option = self.app_schema[name]
|
||||
except KeyError:
|
||||
raw_option = UNKNOWN_OPTION
|
||||
option = OPTION_DEFAULTS.copy()
|
||||
option.update(raw_option)
|
||||
return option
|
||||
|
||||
|
||||
GALAXY_APP = App(
|
||||
["universe_wsgi.ini", "config/galaxy.ini"],
|
||||
"8080",
|
||||
["galaxy.web.buildapp:app_factory"], # TODO: Galaxy could call factory a few different things and they'd all be fine.
|
||||
"config/galaxy.yml",
|
||||
"lib/galaxy/webapps/galaxy/config_schema.yml",
|
||||
'galaxy.webapps.galaxy.buildapp:uwsgi_app()',
|
||||
)
|
||||
SHED_APP = App(
|
||||
["tool_shed_wsgi.ini", "config/tool_shed.ini"],
|
||||
"9009",
|
||||
["galaxy.webapps.tool_shed.buildapp:app_factory"],
|
||||
"config/tool_shed.yml",
|
||||
"lib/galaxy/webapps/tool_shed/config_schema.yml",
|
||||
'galaxy.webapps.tool_shed.buildapp:uwsgi_app()',
|
||||
)
|
||||
REPORTS_APP = App(
|
||||
["reports_wsgi.ini", "config/reports.ini"],
|
||||
"9001",
|
||||
["galaxy.webapps.reports.buildapp:app_factory"],
|
||||
"config/reports.yml",
|
||||
"lib/galaxy/webapps/reports/config_schema.yml",
|
||||
'galaxy.webapps.reports.buildapp:uwsgi_app()',
|
||||
)
|
||||
APPS = {"galaxy": GALAXY_APP, "tool_shed": SHED_APP, "reports": REPORTS_APP}
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
"""Entry point for conversion process."""
|
||||
if argv is None:
|
||||
argv = sys.argv[1:]
|
||||
args = _arg_parser().parse_args(argv)
|
||||
app_name = args.app
|
||||
app_desc = APPS.get(app_name, None)
|
||||
action = args.action
|
||||
action_func = ACTIONS[action]
|
||||
action_func(args, app_desc)
|
||||
|
||||
|
||||
def _arg_parser():
|
||||
parser = argparse.ArgumentParser(description=DESCRIPTION)
|
||||
parser.add_argument('action', metavar='ACTION', type=str,
|
||||
choices=ACTIONS.keys(),
|
||||
help='action to perform')
|
||||
parser.add_argument('app', metavar='APP', type=str, nargs="?",
|
||||
help=APP_DESCRIPTION)
|
||||
parser.add_argument('--add-comments', default=False, action="store_true")
|
||||
parser.add_argument('--dry-run', default=False, action="store_true",
|
||||
help=DRY_RUN_DESCRIPTION)
|
||||
parser.add_argument('--galaxy_root', default=".", type=str)
|
||||
return parser
|
||||
|
||||
|
||||
def _to_rst(args, app_desc, heading_level="~"):
|
||||
rst = StringIO()
|
||||
schema = app_desc.schema
|
||||
for key, value in schema.app_schema.items():
|
||||
default = None if "default" not in value else value["default"]
|
||||
option = schema.get_app_option(key)
|
||||
option_value = OptionValue(key, default, option)
|
||||
_write_option_rst(args, rst, key, heading_level, option_value)
|
||||
print(rst.getvalue())
|
||||
|
||||
|
||||
def _write_option_rst(args, rst, key, heading_level, option_value):
|
||||
title = "``%s``" % key
|
||||
heading = heading_level * len(title)
|
||||
rst.write("%s\n%s\n%s\n\n" % (heading, title, heading))
|
||||
option, value = _parse_option_value(option_value)
|
||||
desc = option["desc"]
|
||||
rst.write(":Description:\n")
|
||||
rst.write("\n".join(RST_DESCRIPTION_WRAPPER.wrap(desc)))
|
||||
rst.write("\n")
|
||||
type = option.get("type", None)
|
||||
default = option.get("default", "*null*")
|
||||
rst.write(":Default: %s\n" % default)
|
||||
if type:
|
||||
rst.write(":Type: %s\n" % type)
|
||||
rst.write("\n\n")
|
||||
|
||||
|
||||
def _build_uwsgi_schema(args, app_desc):
|
||||
req = urllib2.Request('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst')
|
||||
response = urllib2.urlopen(req)
|
||||
rst_options = response.read()
|
||||
last_line = None
|
||||
current_opt = None
|
||||
|
||||
options = OrderedDict({})
|
||||
option = None
|
||||
for line in rst_options.splitlines():
|
||||
line = line.strip()
|
||||
dots = "*" * len(line)
|
||||
if line and (line == dots):
|
||||
current_opt = last_line
|
||||
option = {
|
||||
'type': 'any',
|
||||
}
|
||||
options[current_opt] = option
|
||||
|
||||
if line.startswith("``parser``"):
|
||||
parser = line.split(":", 1)[1].strip()
|
||||
if parser == "uwsgi_opt_set_int":
|
||||
option["type"] = "int"
|
||||
# TODO: disptch on parser...
|
||||
elif line.startswith("``help``"):
|
||||
option["desc"] = line.split(":", 1)[1]
|
||||
|
||||
last_line = line
|
||||
schema = {
|
||||
"type": "map",
|
||||
"desc": "uwsgi definition, see http://uwsgi-docs.readthedocs.io/en/latest/Options.html",
|
||||
"mapping": options
|
||||
}
|
||||
path = os.path.join(args.galaxy_root, UWSGI_SCHEMA_PATH)
|
||||
contents = _ordered_dump(schema)
|
||||
_write_to_file(args, contents, path)
|
||||
|
||||
|
||||
def _find_config(args, app_desc):
|
||||
path = os.path.join(args.galaxy_root, app_desc.destination)
|
||||
if not os.path.exists(path):
|
||||
path = None
|
||||
|
||||
for possible_ini_config_rel in app_desc.config_paths:
|
||||
possible_ini_config = os.path.join(args.galaxy_root, possible_ini_config_rel)
|
||||
if os.path.exists(possible_ini_config):
|
||||
path = possible_ini_config
|
||||
|
||||
if path is None:
|
||||
_warn(USING_SAMPLE_MESSAGE % path)
|
||||
path = os.path.join(args.galaxy_root, app_desc.sample_destination)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def _find_app_options(app_desc, path):
|
||||
"""Load app (as opposed to server) options from specified path.
|
||||
|
||||
Supplied ``path`` may be either YAML or ini file.
|
||||
"""
|
||||
if _is_ini(path):
|
||||
p = nice_config_parser(path)
|
||||
app_items = _find_app_options_from_config_parser(p)
|
||||
else:
|
||||
raw_config = _order_load_path(path)
|
||||
app_items = raw_config.get(app_desc.app_name, None) or {}
|
||||
return app_items
|
||||
|
||||
|
||||
def _find_app_options_from_config_parser(p):
|
||||
if not p.has_section("app:main"):
|
||||
_warn(NO_APP_MAIN_MESSAGE)
|
||||
app_items = OrderedDict()
|
||||
else:
|
||||
app_items = OrderedDict(p.items("app:main"))
|
||||
|
||||
return app_items
|
||||
|
||||
|
||||
def _lint(args, app_desc):
|
||||
path = _find_config(args, app_desc)
|
||||
if not os.path.exists(path):
|
||||
raise Exception("Expected configuration file [%s] not found." % path)
|
||||
app_items = _find_app_options(app_desc, path)
|
||||
for key, value in app_items.items():
|
||||
option_action = OPTION_ACTIONS.get(key)
|
||||
if option_action is not None:
|
||||
option_action.lint(args, app_desc, key, value)
|
||||
|
||||
|
||||
def _validate(args, app_desc):
|
||||
path = _find_config(args, app_desc)
|
||||
with open(path, "r") as f:
|
||||
# Allow empty mapping (not allowed by pykawlify)
|
||||
raw_config = _order_load_path(f)
|
||||
if raw_config.get(app_desc.app_name, None) is None:
|
||||
raw_config[app_desc.app_name] = {}
|
||||
config_p = tempfile.NamedTemporaryFile(delete=False, suffix=".yml")
|
||||
_ordered_dump(raw_config, config_p)
|
||||
config_p.flush()
|
||||
path = config_p.name
|
||||
|
||||
fp = tempfile.NamedTemporaryFile(delete=False, suffix=".yml")
|
||||
_ordered_dump(app_desc.schema.raw_schema, fp)
|
||||
fp.flush()
|
||||
name = fp.name
|
||||
if Core is None:
|
||||
raise Exception("Cannot validate file, pykwalify is not installed.")
|
||||
c = Core(
|
||||
source_file=path,
|
||||
schema_files=[name],
|
||||
)
|
||||
c.validate()
|
||||
|
||||
|
||||
class PrefixFilter(object):
|
||||
|
||||
def __init__(self, name, prefix):
|
||||
self.name = name
|
||||
self.prefix = prefix
|
||||
|
||||
|
||||
class GzipFilter(object):
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
|
||||
def _run_conversion(args, app_desc):
|
||||
ini_config = _find_config(args, app_desc)
|
||||
if ini_config and not _is_ini(ini_config):
|
||||
_warn("Cannot convert YAML file %s, this option is only for ini config files." % ini_config)
|
||||
sys.exit(1)
|
||||
elif not ini_config:
|
||||
_warn("Failed to find a config to convert - exiting without changes.")
|
||||
sys.exit(1)
|
||||
|
||||
p = nice_config_parser(ini_config)
|
||||
server_section = None
|
||||
filters = {}
|
||||
for section in p.sections():
|
||||
if section.startswith("server:"):
|
||||
if server_section:
|
||||
_warn(EXTRA_SERVER_MESSAGE % (server_section, section))
|
||||
else:
|
||||
server_section = section
|
||||
|
||||
if section.startswith("filter:"):
|
||||
filter_name = section[len("filter:"):]
|
||||
filter_type = p.get(section, "use")
|
||||
if filter_type is None:
|
||||
MISSING_FILTER_TYPE_MESSAGE
|
||||
message = EXTRA_SERVER_MESSAGE % section
|
||||
_warn(message)
|
||||
continue
|
||||
|
||||
if filter_type == "egg:PasteDeploy#prefix":
|
||||
prefix = p.get(section, "prefix")
|
||||
filters[filter_name] = PrefixFilter(filter_name, prefix)
|
||||
elif filter_type == "egg:Paste#gzip":
|
||||
filters[filter_name] = GzipFilter(filter_name)
|
||||
else:
|
||||
message = UNHANDLED_FILTER_TYPE_MESSAGE % (filter_type, section)
|
||||
_warn(message)
|
||||
continue
|
||||
|
||||
if not server_section:
|
||||
_warn("No server section found, using default uwsgi server definition.")
|
||||
server_config = OrderedDict()
|
||||
else:
|
||||
server_config = OrderedDict(p.items(server_section))
|
||||
|
||||
app_items = _find_app_options_from_config_parser(p)
|
||||
applied_filters = []
|
||||
if filters:
|
||||
for key, value in app_items.items():
|
||||
if key == "filter-with":
|
||||
if value in filters:
|
||||
applied_filters.append(filters[value])
|
||||
else:
|
||||
_warn("Unknown filter found [%s], exiting..." % value)
|
||||
sys.exit(1)
|
||||
|
||||
uwsgi_dict = _server_paste_to_uwsgi(app_desc, server_config, applied_filters)
|
||||
|
||||
app_dict = OrderedDict({})
|
||||
schema = app_desc.schema
|
||||
for key, value in app_items.items():
|
||||
if key in ["__file__", "here"]:
|
||||
continue
|
||||
|
||||
if key in OPTION_ACTIONS:
|
||||
option_action = OPTION_ACTIONS.get(key)
|
||||
value = option_action.converted(args, app_desc, key, value)
|
||||
|
||||
if value is DROP_OPTION_VALUE:
|
||||
continue
|
||||
|
||||
option = schema.get_app_option(key)
|
||||
if option["unknown_option"]:
|
||||
_warn(UNKNOWN_OPTION_MESSAGE % key)
|
||||
|
||||
option_value = OptionValue(key, value, option)
|
||||
app_dict[key] = option_value
|
||||
|
||||
f = StringIO()
|
||||
_write_section(args, f, "uwsgi", uwsgi_dict)
|
||||
_write_section(args, f, app_desc.app_name, app_dict)
|
||||
destination = os.path.join(args.galaxy_root, app_desc.destination)
|
||||
_replace_file(args, f, app_desc, ini_config, destination)
|
||||
|
||||
|
||||
def _is_ini(path):
|
||||
return path.endswith(".ini") or path.endswith(".ini.sample")
|
||||
|
||||
|
||||
def _replace_file(args, f, app_desc, from_path, to_path):
|
||||
_write_to_file(args, f, to_path)
|
||||
backup_path = "%s.backup" % from_path
|
||||
print("Moving [%s] to [%s]" % (from_path, backup_path))
|
||||
if args.dry_run:
|
||||
print("... skipping because --dry-run is enabled.")
|
||||
else:
|
||||
shutil.move(from_path, backup_path)
|
||||
|
||||
|
||||
def _build_sample_yaml(args, app_desc):
|
||||
schema = app_desc.schema
|
||||
f = StringIO()
|
||||
options = copy.deepcopy(UWSGI_OPTIONS)
|
||||
for key, value in options.items():
|
||||
for field in ["desc", "default"]:
|
||||
if field not in value:
|
||||
continue
|
||||
field_value = value[field]
|
||||
if not isinstance(field_value, six.string_types):
|
||||
continue
|
||||
|
||||
new_field_value = string.Template(field_value).safe_substitute(**{
|
||||
'default_port': str(app_desc.default_port),
|
||||
'app_name': app_desc.app_name,
|
||||
'uwsgi_module': app_desc.uwsgi_module,
|
||||
})
|
||||
value[field] = new_field_value
|
||||
_write_sample_section(args, f, 'uwsgi', Schema(options), as_comment=False)
|
||||
_write_sample_section(args, f, app_desc.app_name, schema)
|
||||
destination = os.path.join(args.galaxy_root, app_desc.sample_destination)
|
||||
_write_to_file(args, f, destination)
|
||||
|
||||
|
||||
def _write_to_file(args, f, path):
|
||||
dry_run = args.dry_run
|
||||
if hasattr(f, "getvalue"):
|
||||
contents = f.getvalue()
|
||||
else:
|
||||
contents = f
|
||||
contents_indented = "\n".join([" |%s" % l for l in contents.splitlines()])
|
||||
print("Writing the file contents:\n%s\nto %s" % (contents_indented, path))
|
||||
if dry_run:
|
||||
print("... skipping because --dry-run is enabled.")
|
||||
else:
|
||||
safe_makedirs(os.path.dirname(path))
|
||||
with open(path, "w") as to_f:
|
||||
to_f.write(contents)
|
||||
|
||||
|
||||
def _order_load_path(path):
|
||||
"""Load (with ``_ordered_load``) on specified path (a YAML file)."""
|
||||
with open(path, "r") as f:
|
||||
# Allow empty mapping (not allowed by pykawlify)
|
||||
raw_config = _ordered_load(f)
|
||||
return raw_config
|
||||
|
||||
|
||||
def _write_sample_section(args, f, section_header, schema, as_comment=True):
|
||||
_write_header(f, section_header)
|
||||
for key, value in schema.app_schema.items():
|
||||
default = None if "default" not in value else value["default"]
|
||||
option = schema.get_app_option(key)
|
||||
option_value = OptionValue(key, default, option)
|
||||
_write_option(args, f, key, option_value, as_comment=as_comment)
|
||||
|
||||
|
||||
def _write_section(args, f, section_header, section_dict):
|
||||
_write_header(f, section_header)
|
||||
for key, option_value in section_dict.items():
|
||||
_write_option(args, f, key, option_value)
|
||||
|
||||
|
||||
def _write_header(f, section_header):
|
||||
f.write("%s:\n\n" % section_header)
|
||||
|
||||
|
||||
def _write_option(args, f, key, option_value, as_comment=False):
|
||||
option, value = _parse_option_value(option_value)
|
||||
desc = option["desc"]
|
||||
comment = ""
|
||||
if desc and args.add_comments:
|
||||
comment = "\n".join(YAML_COMMENT_WRAPPER.wrap(desc))
|
||||
comment += "\n"
|
||||
as_comment_str = "#" if as_comment else ""
|
||||
key_val_str = yaml.dump({key: value}).lstrip("{").rstrip("\n}")
|
||||
lines = "%s%s%s" % (comment, as_comment_str, key_val_str)
|
||||
lines_idented = "\n".join([(" %s" % l) for l in lines.split("\n")])
|
||||
f.write("%s\n\n" % lines_idented)
|
||||
|
||||
|
||||
def _parse_option_value(option_value):
|
||||
if isinstance(option_value, OptionValue):
|
||||
option = option_value.option
|
||||
value = option_value.value
|
||||
else:
|
||||
value = option_value
|
||||
option = OPTION_DEFAULTS
|
||||
return option, value
|
||||
|
||||
|
||||
def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
|
||||
uwsgi_dict = OrderedDict()
|
||||
port = server_config.get("port", app_desc.default_port)
|
||||
host = server_config.get("host", "127.0.0.1")
|
||||
|
||||
if server_config.get("use", "egg:Paste#http") != "egg:Paste#http":
|
||||
raise Exception("Unhandled paste server 'use' value [%s], file must be manually migrate.")
|
||||
|
||||
uwsgi_dict["http"] = "%s:%s" % (host, port)
|
||||
# default changing from 10 to 8
|
||||
uwsgi_dict["threads"] = int(server_config.get("threadpool_workers", 8))
|
||||
# required for static...
|
||||
uwsgi_dict["http-raw-body"] = True
|
||||
uwsgi_dict["offload-threads"] = 8
|
||||
|
||||
# Handle paste filters during conversion.
|
||||
prefix = None
|
||||
for applied_filter in applied_filters:
|
||||
if isinstance(applied_filter, PrefixFilter):
|
||||
prefix = applied_filter.prefix
|
||||
break
|
||||
elif isinstance(applied_filter, GzipFilter):
|
||||
uwsgi_dict["http-auto-gzip"] = True
|
||||
|
||||
if prefix:
|
||||
uwsgi_dict["mount"] = "%s=%s" % (prefix, app_desc.uwsgi_module)
|
||||
uwsgi_dict["manage-script-name"] = True
|
||||
else:
|
||||
uwsgi_dict["module"] = app_desc.uwsgi_module
|
||||
return uwsgi_dict
|
||||
|
||||
|
||||
def _warn(message):
|
||||
print("WARNING: %s" % message)
|
||||
|
||||
|
||||
def _ordered_load(stream):
|
||||
|
||||
class OrderedLoader(yaml.Loader):
|
||||
|
||||
def __init__(self, stream):
|
||||
self._root = os.path.split(stream.name)[0]
|
||||
super(OrderedLoader, self).__init__(stream)
|
||||
|
||||
def include(self, node):
|
||||
filename = os.path.join(self._root, self.construct_scalar(node))
|
||||
with open(filename, 'r') as f:
|
||||
return yaml.load(f, OrderedLoader)
|
||||
|
||||
def construct_mapping(loader, node):
|
||||
loader.flatten_mapping(node)
|
||||
return OrderedDict(loader.construct_pairs(node))
|
||||
|
||||
OrderedLoader.add_constructor(
|
||||
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
|
||||
construct_mapping)
|
||||
OrderedLoader.add_constructor('!include', OrderedLoader.include)
|
||||
|
||||
return yaml.load(stream, OrderedLoader)
|
||||
|
||||
|
||||
def _ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds):
|
||||
class OrderedDumper(Dumper):
|
||||
pass
|
||||
|
||||
def _dict_representer(dumper, data):
|
||||
return dumper.represent_mapping(
|
||||
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
|
||||
data.items())
|
||||
OrderedDumper.add_representer(OrderedDict, _dict_representer)
|
||||
return yaml.dump(data, stream, OrderedDumper, **kwds)
|
||||
|
||||
|
||||
ACTIONS = {
|
||||
"convert": _run_conversion,
|
||||
"build_sample_yaml": _build_sample_yaml,
|
||||
"validate": _validate,
|
||||
"lint": _lint,
|
||||
"build_uwsgi_yaml": _build_uwsgi_schema,
|
||||
"build_rst": _to_rst,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -32,6 +32,8 @@ from galaxy.workflow.modules import module_factory
|
||||
from galaxy.workflow.run import invoke, queue_invoke
|
||||
from galaxy.workflow.run_request import build_workflow_run_configs
|
||||
|
||||
from tool_shed.galaxy_install.install_manager import InstallRepositoryManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -500,6 +502,10 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
|
||||
def __api_import_new_workflow( self, trans, payload, **kwd ):
|
||||
data = payload['workflow']
|
||||
|
||||
import_tools = util.string_as_bool( payload.get( "import_tools", False ) )
|
||||
if import_tools and not trans.user_is_admin():
|
||||
raise exceptions.AdminRequiredException()
|
||||
|
||||
publish = util.string_as_bool( payload.get( "publish", False ) )
|
||||
# If 'publish' set, default to importable.
|
||||
importable = util.string_as_bool( payload.get( "importable", publish ) )
|
||||
@@ -534,6 +540,32 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
|
||||
|
||||
rval.append(item)
|
||||
|
||||
#
|
||||
if import_tools:
|
||||
tools = {}
|
||||
for key in data['steps']:
|
||||
item = data['steps'][key]
|
||||
if item is not None:
|
||||
if 'tool_shed_repository' in item:
|
||||
tool_shed_repository = item['tool_shed_repository']
|
||||
if 'owner' in tool_shed_repository and 'changeset_revision' in tool_shed_repository and 'name' in tool_shed_repository and 'tool_shed' in tool_shed_repository:
|
||||
toolstr = tool_shed_repository['owner'] \
|
||||
+ tool_shed_repository['changeset_revision'] \
|
||||
+ tool_shed_repository['name'] \
|
||||
+ tool_shed_repository['tool_shed']
|
||||
tools[toolstr] = tool_shed_repository
|
||||
irm = InstallRepositoryManager( self.app )
|
||||
for k in tools:
|
||||
item = tools[k]
|
||||
tool_shed_url = 'https://' + item['tool_shed'] + '/'
|
||||
name = item['name']
|
||||
owner = item['owner']
|
||||
changeset_revision = item['changeset_revision']
|
||||
irm.install( tool_shed_url,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision,
|
||||
payload )
|
||||
return item
|
||||
|
||||
@expose_api
|
||||
|
||||
@@ -160,6 +160,8 @@ def paste_app_factory( global_conf, **kwargs ):
|
||||
|
||||
|
||||
def uwsgi_app_factory():
|
||||
# TODO: synchronize with galaxy.web.framework.webapp.build_native_uwsgi_app - should
|
||||
# at least be using nice_config_parser for instance.
|
||||
import uwsgi
|
||||
root = os.path.abspath(uwsgi.opt.get('galaxy_root', os.getcwd()))
|
||||
config_file = uwsgi.opt.get('galaxy_config_file', os.path.join(root, 'config', 'galaxy.ini'))
|
||||
|
||||
@@ -78,24 +78,21 @@ class User( BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Creat
|
||||
if not trans.app.config.enable_openid:
|
||||
return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' )
|
||||
message = 'Unspecified failure authenticating via OpenID'
|
||||
openid_url = kwd.get( 'openid_url', '' )
|
||||
openid_provider = kwd.get( 'openid_provider', '' )
|
||||
if not openid_provider or openid_url:
|
||||
openid_provider = trans.app.openid_providers.NO_PROVIDER_ID # empty fields cause validation errors
|
||||
redirect = kwd.get( 'redirect', '' ).strip()
|
||||
auto_associate = util.string_as_bool( kwd.get( 'auto_associate', False ) )
|
||||
use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) )
|
||||
action = 'login'
|
||||
consumer = trans.app.openid_manager.get_consumer( trans )
|
||||
openid_url = kwd.get( 'openid_url', '' )
|
||||
openid_provider = kwd.get('openid_provider', '')
|
||||
if openid_url:
|
||||
openid_provider_obj = trans.app.openid_providers.new_provider_from_identifier( openid_url )
|
||||
elif openid_provider:
|
||||
openid_provider_obj = trans.app.openid_providers.get(openid_provider)
|
||||
else:
|
||||
openid_provider_obj = trans.app.openid_providers.get( openid_provider )
|
||||
if not openid_url and openid_provider == trans.app.openid_providers.NO_PROVIDER_ID:
|
||||
message = 'An OpenID provider was not specified'
|
||||
elif openid_provider_obj:
|
||||
if not redirect:
|
||||
redirect = ' '
|
||||
redirect = kwd.get('redirect', '').strip()
|
||||
if not redirect:
|
||||
redirect = ' '
|
||||
if openid_provider_obj:
|
||||
process_url = trans.request.base.rstrip( '/' ) + url_for( controller='user', action='openid_process', redirect=redirect, openid_provider=openid_provider, auto_associate=auto_associate ) # None of these values can be empty, or else a verification error will occur
|
||||
request = None
|
||||
try:
|
||||
@@ -116,7 +113,7 @@ class User( BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Creat
|
||||
trans.app.openid_manager.persist_session( trans, consumer )
|
||||
return form
|
||||
return trans.response.send_redirect( url_for( controller='user',
|
||||
action=action,
|
||||
action='login',
|
||||
redirect=redirect,
|
||||
use_panels=use_panels,
|
||||
message=message,
|
||||
|
||||
@@ -51,11 +51,12 @@ def add_ui_controllers( webapp, app ):
|
||||
webapp.add_ui_controller( name, T( app ) )
|
||||
|
||||
|
||||
def app_factory( global_conf, **kwargs ):
|
||||
def app_factory( global_conf, load_app_kwds={}, **kwargs ):
|
||||
"""Return a wsgi application serving the root object"""
|
||||
# Create the Galaxy application unless passed in
|
||||
kwargs = load_app_properties(
|
||||
kwds=kwargs
|
||||
kwds=kwargs,
|
||||
**load_app_kwds
|
||||
)
|
||||
if 'app' in kwargs:
|
||||
app = kwargs.pop( 'app' )
|
||||
@@ -149,3 +150,7 @@ def wrap_in_middleware( app, global_conf, application_stack, **local_conf ):
|
||||
def wrap_in_static( app, global_conf, **local_conf ):
|
||||
urlmap, _ = galaxy.web.framework.webapp.build_url_map( app, global_conf, local_conf )
|
||||
return urlmap
|
||||
|
||||
|
||||
def uwsgi_app():
|
||||
return galaxy.web.framework.webapp.build_native_uwsgi_app( app_factory, "reports" )
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
type: map
|
||||
mapping:
|
||||
uwsgi: !include uwsgi_schema.yml
|
||||
reports:
|
||||
type: map
|
||||
required: true
|
||||
desc: |
|
||||
Galaxy Reports configuration options.
|
||||
mapping:
|
||||
filter-with:
|
||||
type: str
|
||||
default: proxy-prefix
|
||||
desc: |
|
||||
If running behind a proxy server and Galaxy is served from a subdirectory,
|
||||
enable the proxy-prefix filter and set the prefix in the
|
||||
[filter:proxy-prefix] section above.
|
||||
|
||||
cookie_path:
|
||||
type: str
|
||||
desc: |
|
||||
If proxy-prefix is enabled and you're running more than one Galaxy instance
|
||||
behind one hostname, you will want to set this to the same path as the prefix
|
||||
in the filter above. This value becomes the "path" attribute set in the
|
||||
cookie so the cookies from each instance will not clobber each other.
|
||||
|
||||
log_level:
|
||||
type: str
|
||||
default: DEBUG
|
||||
desc: |
|
||||
Verbosity of console log messages. Acceptable values can be found here:
|
||||
https://docs.python.org/2/library/logging.html#logging-levels
|
||||
|
||||
database_connection:
|
||||
type: str
|
||||
default: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
|
||||
desc: |
|
||||
Database connection
|
||||
Galaxy reports are intended for production Galaxy instances, so sqlite (and the default value
|
||||
below) is not supported. An SQLAlchemy connection string should be used specify an external
|
||||
database.
|
||||
|
||||
file_path:
|
||||
type: str
|
||||
default: database/files
|
||||
desc: |
|
||||
Where dataset files are saved
|
||||
Temporary storage for additional datasets, this should be shared through the cluster
|
||||
|
||||
new_file_path:
|
||||
type: str
|
||||
default: database/tmp
|
||||
desc: |
|
||||
Where dataset files are saved
|
||||
Temporary storage for additional datasets, this should be shared through the cluster
|
||||
|
||||
template_cache_path:
|
||||
type: str
|
||||
default: database/compiled_templates/reports
|
||||
desc: |
|
||||
Mako templates are compiled as needed and cached for reuse, this directory is
|
||||
used for the cache
|
||||
|
||||
debug:
|
||||
type: bool
|
||||
default: false
|
||||
desc: |
|
||||
Configuration for debugging middleware
|
||||
|
||||
use_lint:
|
||||
type: bool
|
||||
default: false
|
||||
desc: |
|
||||
Check for WSGI compliance.
|
||||
|
||||
use_interactive:
|
||||
type: bool
|
||||
default: true
|
||||
desc: |
|
||||
NEVER enable this on a public site (even test or QA)
|
||||
|
||||
use_heartbeat:
|
||||
type: bool
|
||||
default: true
|
||||
desc: |
|
||||
Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!)
|
||||
|
||||
use_profile:
|
||||
type: bool
|
||||
default: true
|
||||
desc: |
|
||||
Profiling middleware (cProfile based)
|
||||
|
||||
smtp_server:
|
||||
type: str
|
||||
default: yourserver@yourfacility.edu
|
||||
desc: |
|
||||
Mail
|
||||
|
||||
error_email_to:
|
||||
type: str
|
||||
default: your_bugs@bx.psu.edu
|
||||
desc: |
|
||||
Mail
|
||||
@@ -0,0 +1 @@
|
||||
../uwsgi_schema.yml
|
||||
@@ -53,12 +53,13 @@ def add_ui_controllers( webapp, app ):
|
||||
webapp.add_ui_controller( name, T( app ) )
|
||||
|
||||
|
||||
def app_factory( global_conf, **kwargs ):
|
||||
def app_factory( global_conf, load_app_kwds={}, **kwargs ):
|
||||
"""Return a wsgi application serving the root object"""
|
||||
# Create the Galaxy tool shed application unless passed in
|
||||
kwargs = load_app_properties(
|
||||
kwds=kwargs,
|
||||
config_prefix='TOOL_SHED_CONFIG_'
|
||||
config_prefix='TOOL_SHED_CONFIG_',
|
||||
**load_app_kwds
|
||||
)
|
||||
if 'app' in kwargs:
|
||||
app = kwargs.pop( 'app' )
|
||||
@@ -309,3 +310,7 @@ def _map_redirects( mapper ):
|
||||
|
||||
mapper.redirect( "/repository/status_for_installed_repository", "/api/repositories/updates/", _redirect_code="301 Moved Permanently", conditions=dict( function=forward_qs ) )
|
||||
return mapper
|
||||
|
||||
|
||||
def uwsgi_app():
|
||||
return galaxy.web.framework.webapp.build_native_uwsgi_app( app_factory, "tool_shed" )
|
||||
|
||||
@@ -0,0 +1,387 @@
|
||||
type: map
|
||||
mapping:
|
||||
uwsgi: !include uwsgi_schema.yml
|
||||
tool_shed:
|
||||
type: map
|
||||
required: true
|
||||
desc: |
|
||||
Tool Shed configuration options.
|
||||
mapping:
|
||||
log_level:
|
||||
type: str
|
||||
default: DEBUG
|
||||
required: false
|
||||
desc: |
|
||||
Verbosity of console log messages. Acceptable values can be found here:
|
||||
https://docs.python.org/2/library/logging.html#logging-levels
|
||||
|
||||
database_connection:
|
||||
type: str
|
||||
default: sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE
|
||||
required: false
|
||||
desc: |
|
||||
By default, the Tool Shed uses a SQLite database at 'database/community.sqlite'. You
|
||||
may use a SQLAlchemy connection string to specify an external database
|
||||
instead. This string takes many options which are explained in detail in the
|
||||
config file documentation.
|
||||
|
||||
hgweb_config_dir:
|
||||
type: str
|
||||
required: false
|
||||
desc: |
|
||||
Where the hgweb.config file is stored.
|
||||
The default is the Galaxy installation directory.
|
||||
|
||||
file_path:
|
||||
type: str
|
||||
default: database/community_files
|
||||
required: false
|
||||
desc: |
|
||||
Where tool shed repositories are stored.
|
||||
|
||||
new_file_path:
|
||||
type: str
|
||||
default: database/tmp
|
||||
required: false
|
||||
desc: |
|
||||
Temporary storage for additional datasets,
|
||||
this should be shared through the cluster
|
||||
|
||||
builds_file_path:
|
||||
type: str
|
||||
default: tool-data/shared/ucsc/builds.txt
|
||||
required: false
|
||||
desc: |
|
||||
File containing old-style genome builds
|
||||
|
||||
pretty_datetime_format:
|
||||
type: str
|
||||
default: $locale (UTC)
|
||||
required: false
|
||||
desc: |
|
||||
Format string used when showing date and time information.
|
||||
The string may contain:
|
||||
- the directives used by Python time.strftime() function (see
|
||||
https://docs.python.org/2/library/time.html#time.strftime ),
|
||||
- $locale (complete format string for the server locale),
|
||||
- $iso8601 (complete format string as specified by ISO 8601 international
|
||||
standard).
|
||||
|
||||
toolshed_search_on:
|
||||
type: bool
|
||||
default: true
|
||||
required: false
|
||||
desc: |
|
||||
-- Repository and Tool search
|
||||
Using the script located at scripts/build_ts_whoosh_index.py
|
||||
you can generate search index and allow full text API searching over
|
||||
the repositories and tools within the Tool Shed given that you specify
|
||||
the following two config options.
|
||||
|
||||
whoosh_index_dir:
|
||||
type: str
|
||||
default: database/toolshed_whoosh_indexes
|
||||
required: false
|
||||
desc: |
|
||||
-- Repository and Tool search
|
||||
Using the script located at scripts/build_ts_whoosh_index.py
|
||||
you can generate search index and allow full text API searching over
|
||||
the repositories and tools within the Tool Shed given that you specify
|
||||
the following two config options.
|
||||
|
||||
repo_name_boost:
|
||||
type: float
|
||||
default: 0.9
|
||||
required: false
|
||||
desc: |
|
||||
For searching repositories at /api/repositories:
|
||||
|
||||
repo_description_boost:
|
||||
type: float
|
||||
default: 0.6
|
||||
required: false
|
||||
desc: |
|
||||
For searching repositories at /api/repositories:
|
||||
|
||||
repo_long_description_boost:
|
||||
type: float
|
||||
default: 0.5
|
||||
required: false
|
||||
desc: |
|
||||
For searching repositories at /api/repositories:
|
||||
|
||||
repo_homepage_url_boost:
|
||||
type: float
|
||||
default: 0.3
|
||||
required: false
|
||||
desc: |
|
||||
For searching repositories at /api/repositories:
|
||||
|
||||
repo_remote_repository_url_boost:
|
||||
type: float
|
||||
default: 0.2
|
||||
required: false
|
||||
desc: |
|
||||
For searching repositories at /api/repositories:
|
||||
|
||||
repo_owner_username_boost:
|
||||
type: float
|
||||
default: 0.3
|
||||
required: false
|
||||
desc: |
|
||||
For searching repositories at /api/repositories:
|
||||
|
||||
tool_name_boost:
|
||||
type: float
|
||||
default: 1.2
|
||||
required: false
|
||||
desc: |
|
||||
For searching tools at /api/tools
|
||||
|
||||
tool_description_boost:
|
||||
type: float
|
||||
default: 0.6
|
||||
required: false
|
||||
desc: |
|
||||
For searching tools at /api/tools
|
||||
|
||||
tool_help_boost:
|
||||
type: float
|
||||
default: 0.4
|
||||
required: false
|
||||
desc: |
|
||||
For searching tools at /api/tools
|
||||
|
||||
tool_repo_owner_username:
|
||||
type: float
|
||||
default: 0.3
|
||||
required: false
|
||||
desc: |
|
||||
For searching tools at /api/tools
|
||||
|
||||
ga_code:
|
||||
type: str
|
||||
required: false
|
||||
desc: |
|
||||
You can enter tracking code here to track visitor's behavior
|
||||
through your Google Analytics account. Example: UA-XXXXXXXX-Y
|
||||
|
||||
id_secret:
|
||||
type: str
|
||||
default: changethisinproductiontoo
|
||||
required: false
|
||||
desc: |
|
||||
The Tool Shed encodes various internal values when these values will be output in
|
||||
some format (for example, in a URL or cookie). You should set a key to be
|
||||
used by the algorithm that encodes and decodes these values. It can be any
|
||||
string. If left unchanged, anyone could construct a cookie that would grant
|
||||
them access to others' sessions.
|
||||
One simple way to generate a value for this is with the shell command:
|
||||
python -c 'import time; print time.time()' | md5sum | cut -f 1 -d ' '
|
||||
|
||||
use_remote_user:
|
||||
type: bool
|
||||
default: false
|
||||
required: false
|
||||
desc: |
|
||||
User authentication can be delegated to an upstream proxy server (usually
|
||||
Apache). The upstream proxy should set a REMOTE_USER header in the request.
|
||||
Enabling remote user disables regular logins. For more information, see:
|
||||
https://wiki.galaxyproject.org/Admin/Config/ApacheProxy
|
||||
|
||||
remote_user_secret:
|
||||
type: str
|
||||
default: changethisinproductiontoo
|
||||
required: false
|
||||
desc: |
|
||||
If use_remote_user is enabled, anyone who can log in to the Galaxy host may
|
||||
impersonate any other user by simply sending the appropriate header. Thus a
|
||||
secret shared between the upstream proxy server, and Galaxy is required.
|
||||
If anyone other than the Galaxy user is using the server, then apache/nginx
|
||||
should pass a value in the header 'GX_SECRET' that is identical the one below
|
||||
|
||||
debug:
|
||||
type: bool
|
||||
default: false
|
||||
required: false
|
||||
desc: |
|
||||
Configuration for debugging middleware
|
||||
|
||||
use_lint:
|
||||
type: bool
|
||||
default: false
|
||||
required: false
|
||||
desc: |
|
||||
Check for WSGI compliance.
|
||||
|
||||
use_printdebug:
|
||||
type: bool
|
||||
default: true
|
||||
required: false
|
||||
desc: |
|
||||
Intercept print statements and show them on the returned page.
|
||||
|
||||
use_interactive:
|
||||
type: bool
|
||||
default: true
|
||||
required: false
|
||||
desc: |
|
||||
NEVER enable this on a public site (even test or QA)
|
||||
|
||||
admin_users:
|
||||
type: str
|
||||
required: false
|
||||
desc: |
|
||||
Administrative users - set this to a comma-separated list of valid Tool Shed
|
||||
users (email addresses). These users will have access to the Admin section
|
||||
of the server, and will have access to create users, groups, roles,
|
||||
libraries, and more.
|
||||
|
||||
require_login:
|
||||
type: bool
|
||||
default: false
|
||||
required: false
|
||||
desc: |
|
||||
Force everyone to log in (disable anonymous access)
|
||||
|
||||
smtp_server:
|
||||
type: str
|
||||
default: smtp.your_tool_shed_server
|
||||
required: false
|
||||
desc: |
|
||||
For use by email messages sent from the tool shed
|
||||
|
||||
email_from:
|
||||
type: str
|
||||
default: your_tool_shed_email@server
|
||||
required: false
|
||||
desc: |
|
||||
For use by email messages sent from the tool shed
|
||||
|
||||
smtp_username:
|
||||
type: str
|
||||
required: false
|
||||
desc: |
|
||||
If your SMTP server requires a username and password, you can provide them
|
||||
here (password in cleartext here, but if your server supports STARTTLS it
|
||||
will be sent over the network encrypted).
|
||||
|
||||
smtp_password:
|
||||
type: str
|
||||
required: false
|
||||
desc: |
|
||||
If your SMTP server requires a username and password, you can provide them
|
||||
here (password in cleartext here, but if your server supports STARTTLS it
|
||||
will be sent over the network encrypted).
|
||||
|
||||
smtp_ssl:
|
||||
type: bool
|
||||
default: false
|
||||
required: false
|
||||
desc: |
|
||||
If your SMTP server requires SSL from the beginning of the connection
|
||||
|
||||
support_url:
|
||||
type: str
|
||||
default: https://wiki.galaxyproject.org/Support
|
||||
required: false
|
||||
desc: |
|
||||
The URL linked by the "Support" link in the "Help" menu.
|
||||
|
||||
mailing_join_addr:
|
||||
type: str
|
||||
default: galaxy-announce-join@bx.psu.edu
|
||||
required: false
|
||||
desc: |
|
||||
Address to join mailing list
|
||||
|
||||
use_heartbeat:
|
||||
type: bool
|
||||
default: true
|
||||
required: false
|
||||
desc: |
|
||||
Write thread status periodically to 'heartbeat.log' (careful, uses disk
|
||||
space rapidly!)
|
||||
|
||||
use_profile:
|
||||
type: bool
|
||||
default: true
|
||||
required: false
|
||||
desc: |
|
||||
Profiling middleware (cProfile based)
|
||||
|
||||
enable_galaxy_flavor_docker_image:
|
||||
type: bool
|
||||
default: false
|
||||
required: false
|
||||
desc: |
|
||||
Enable creation of Galaxy flavor Docker Image
|
||||
|
||||
message_box_visible:
|
||||
type: bool
|
||||
default: false
|
||||
required: false
|
||||
desc: |
|
||||
Show a message box under the masthead.
|
||||
|
||||
message_box_content:
|
||||
type: str
|
||||
required: false
|
||||
desc: |
|
||||
Show a message box under the masthead.
|
||||
|
||||
message_box_class:
|
||||
type: str
|
||||
default: info
|
||||
required: false
|
||||
desc: |
|
||||
Show a message box under the masthead.
|
||||
|
||||
static_enabled:
|
||||
type: bool
|
||||
default: true
|
||||
required: false
|
||||
desc: |
|
||||
Serving static files (needed if running standalone)
|
||||
|
||||
static_cache_time:
|
||||
type: int
|
||||
default: 360
|
||||
required: false
|
||||
desc: |
|
||||
Serving static files (needed if running standalone)
|
||||
|
||||
static_dir:
|
||||
type: str
|
||||
default: static/
|
||||
required: false
|
||||
desc: |
|
||||
Serving static files (needed if running standalone)
|
||||
|
||||
static_images_dir:
|
||||
type: str
|
||||
default: static/images
|
||||
required: false
|
||||
desc: |
|
||||
Serving static files (needed if running standalone)
|
||||
|
||||
static_favicon_dir:
|
||||
type: str
|
||||
default: static/favicon.ico
|
||||
required: false
|
||||
desc: |
|
||||
Serving static files (needed if running standalone)
|
||||
|
||||
static_scripts_dir:
|
||||
type: str
|
||||
default: static/scripts/
|
||||
required: false
|
||||
desc: |
|
||||
Serving static files (needed if running standalone)
|
||||
|
||||
static_style_dir:
|
||||
type: str
|
||||
default: static/style/blue
|
||||
required: false
|
||||
desc: |
|
||||
Serving static files (needed if running standalone)
|
||||
@@ -0,0 +1 @@
|
||||
../uwsgi_schema.yml
|
||||
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0"?>
|
||||
<provider id="yahoo" name="Yahoo!">
|
||||
<op_endpoint_url>http://yahoo.com</op_endpoint_url>
|
||||
<op_endpoint_url>http://openid.yahoo.com</op_endpoint_url>
|
||||
</provider>
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
. ./scripts/common_startup_functions.sh
|
||||
|
||||
# If there is a file that defines a shell environment specific to this
|
||||
# instance of Galaxy, source the file.
|
||||
if [ -z "$GALAXY_LOCAL_ENV_FILE" ];
|
||||
@@ -14,72 +16,13 @@ then
|
||||
. $GALAXY_LOCAL_ENV_FILE
|
||||
fi
|
||||
|
||||
INITIALIZE_TOOL_DEPENDENCIES=1 # Install Conda if needed.
|
||||
# Pop args meant for common_startup.sh
|
||||
while :
|
||||
do
|
||||
case "$1" in
|
||||
--skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip|--replace-pip)
|
||||
common_startup_args="$common_startup_args $1"
|
||||
shift
|
||||
;;
|
||||
--skip-tool-dependency-initialization)
|
||||
INITIALIZE_TOOL_DEPENDENCIES=0
|
||||
shift
|
||||
;;
|
||||
--skip-venv)
|
||||
skip_venv=1
|
||||
common_startup_args="$common_startup_args $1"
|
||||
shift
|
||||
;;
|
||||
--stop-daemon)
|
||||
common_startup_args="$common_startup_args $1"
|
||||
paster_args="$paster_args $1"
|
||||
stop_daemon_arg_set=1
|
||||
shift
|
||||
;;
|
||||
--daemon|--restart|restart)
|
||||
if [ "$1" = "--restart" ]
|
||||
then
|
||||
paster_args="$paster_args restart"
|
||||
else
|
||||
paster_args="$paster_args $1"
|
||||
fi
|
||||
|
||||
daemon_or_restart_arg_set=1
|
||||
shift
|
||||
;;
|
||||
--wait)
|
||||
wait_arg_set=1
|
||||
shift
|
||||
;;
|
||||
"")
|
||||
break
|
||||
;;
|
||||
*)
|
||||
paster_args="$paster_args $1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
./scripts/common_startup.sh $common_startup_args || exit 1
|
||||
|
||||
# If there is a .venv/ directory, assume it contains a virtualenv that we
|
||||
# should run this instance in.
|
||||
GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-.venv}"
|
||||
if [ -d "$GALAXY_VIRTUAL_ENV" -a -z "$skip_venv" ];
|
||||
then
|
||||
[ -n "$PYTHONPATH" ] && { echo 'Unsetting $PYTHONPATH'; unset PYTHONPATH; }
|
||||
echo "Activating virtualenv at $GALAXY_VIRTUAL_ENV"
|
||||
. "$GALAXY_VIRTUAL_ENV/bin/activate"
|
||||
fi
|
||||
parse_common_args $@
|
||||
|
||||
# If you are using --skip-venv we assume you know what you are doing but warn
|
||||
# in case you don't.
|
||||
[ -n "$PYTHONPATH" ] && echo 'WARNING: $PYTHONPATH is set, this can cause problems importing Galaxy dependencies'
|
||||
run_common_start_up
|
||||
|
||||
python ./scripts/check_python.py || exit 1
|
||||
setup_python
|
||||
|
||||
if [ ! -z "$GALAXY_RUN_WITH_TEST_TOOLS" ];
|
||||
then
|
||||
|
||||
+20
-17
@@ -9,17 +9,27 @@
|
||||
# argument to this will cause Galaxy's database and path parameters
|
||||
# from galaxy.ini to be copied over into reports.ini.
|
||||
|
||||
cd `dirname $0`
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
./scripts/common_startup.sh --skip-samples
|
||||
GALAXY_REPORTS_PID=${GALAXY_REPORTS_PID:-reports_webapp.pid}
|
||||
GALAXY_REPORTS_LOG=${GALAXY_REPORTS_LOG:-reports_webapp.log}
|
||||
PID_FILE=$GALAXY_REPORTS_PID
|
||||
LOG_FILE=$GALAXY_REPORTS_LOG
|
||||
|
||||
: ${GALAXY_VIRTUAL_ENV:=.venv}
|
||||
. ./scripts/common_startup_functions.sh
|
||||
|
||||
if [ -d "$GALAXY_VIRTUAL_ENV" ];
|
||||
if [ "$1" = "--sync-config" ];
|
||||
then
|
||||
. "$GALAXY_VIRTUAL_ENV/bin/activate"
|
||||
python ./scripts/sync_reports_config.py
|
||||
shift
|
||||
fi
|
||||
|
||||
parse_common_args $@
|
||||
|
||||
run_common_start_up
|
||||
|
||||
setup_python
|
||||
|
||||
if [ -z "$GALAXY_REPORTS_CONFIG" ]; then
|
||||
if [ -f reports_wsgi.ini ]; then
|
||||
GALAXY_REPORTS_CONFIG=reports_wsgi.ini
|
||||
@@ -27,24 +37,17 @@ if [ -z "$GALAXY_REPORTS_CONFIG" ]; then
|
||||
GALAXY_REPORTS_CONFIG=config/reports_wsgi.ini
|
||||
elif [ -f config/reports.ini ]; then
|
||||
GALAXY_REPORTS_CONFIG=config/reports.ini
|
||||
elif [ -f config/reports.yml ]; then
|
||||
GALAXY_REPORTS_CONFIG=config/reports.yml
|
||||
else
|
||||
GALAXY_REPORTS_CONFIG=config/reports.ini.sample
|
||||
GALAXY_REPORTS_CONFIG=config/reports.yml.sample
|
||||
fi
|
||||
export GALAXY_REPORTS_CONFIG
|
||||
fi
|
||||
|
||||
GALAXY_REPORTS_PID=${GALAXY_REPORTS_PID:-reports_webapp.pid}
|
||||
GALAXY_REPORTS_LOG=${GALAXY_REPORTS_LOG:-reports_webapp.log}
|
||||
|
||||
if [ -n "$GALAXY_REPORTS_CONFIG_DIR" ]; then
|
||||
python ./scripts/build_universe_config.py "$GALAXY_REPORTS_CONFIG_DIR" "$GALAXY_REPORTS_CONFIG"
|
||||
fi
|
||||
|
||||
|
||||
if [ "$1" = "--sync-config" ];
|
||||
then
|
||||
python ./scripts/sync_reports_config.py
|
||||
shift
|
||||
fi
|
||||
|
||||
python ./scripts/paster.py serve "$GALAXY_REPORTS_CONFIG" --pid-file="$GALAXY_REPORTS_PID" --log-file="$GALAXY_REPORTS_LOG" $@
|
||||
find_server $GALAXY_REPORTS_CONFIG
|
||||
$run_server $server_args
|
||||
|
||||
+23
-22
@@ -1,29 +1,27 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd `dirname $0`
|
||||
|
||||
: ${GALAXY_VIRTUAL_ENV:=.venv}
|
||||
|
||||
if [ -d "$GALAXY_VIRTUAL_ENV" ];
|
||||
then
|
||||
. "$GALAXY_VIRTUAL_ENV/bin/activate"
|
||||
fi
|
||||
|
||||
./scripts/common_startup.sh
|
||||
|
||||
: ${GALAXY_VIRTUAL_ENV:=.venv}
|
||||
|
||||
if [ -d "$GALAXY_VIRTUAL_ENV" ];
|
||||
then
|
||||
. "$GALAXY_VIRTUAL_ENV/bin/activate"
|
||||
fi
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
|
||||
tool_shed=`./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $@`
|
||||
args=$@
|
||||
TOOL_SHED_PID=${TOOL_SHED_PID:-tool_shed_webapp.pid}
|
||||
TOOL_SHED_LOG=${TOOL_SHED_LOG:-tool_shed_webapp.log}
|
||||
PID_FILE=$TOOL_SHED_PID
|
||||
LOG_FILE=$TOOL_SHED_LOG
|
||||
|
||||
. ./scripts/common_startup_functions.sh
|
||||
|
||||
parse_common_args $@
|
||||
|
||||
run_common_start_up
|
||||
|
||||
setup_python
|
||||
|
||||
|
||||
tool_shed=`./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $parser_args`
|
||||
args=$parser_args
|
||||
|
||||
if [ $? -eq 0 ] ; then
|
||||
bash ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_tool_shed.sh $@
|
||||
bash ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_tool_shed.sh $parser_args
|
||||
args=`echo $@ | sed "s#-\?-bootstrap_from_tool_shed $tool_shed##"`
|
||||
fi
|
||||
|
||||
@@ -32,10 +30,13 @@ if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then
|
||||
TOOL_SHED_CONFIG_FILE=tool_shed_wsgi.ini
|
||||
elif [ -f config/tool_shed.ini ]; then
|
||||
TOOL_SHED_CONFIG_FILE=config/tool_shed.ini
|
||||
elif [ -f config/tool_shed.yml ]; then
|
||||
TOOL_SHED_CONFIG_FILE=config/tool_shed.yml
|
||||
else
|
||||
TOOL_SHED_CONFIG_FILE=config/tool_shed.ini.sample
|
||||
TOOL_SHED_CONFIG_FILE=config/tool_shed.yml.sample
|
||||
fi
|
||||
export TOOL_SHED_CONFIG_FILE
|
||||
fi
|
||||
|
||||
python ./scripts/paster.py serve $TOOL_SHED_CONFIG_FILE --pid-file=tool_shed_webapp.pid --log-file=tool_shed_webapp.log $args
|
||||
find_server $TOOL_SHED_CONFIG_FILE
|
||||
$run_server $server_args
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/bin/sh
|
||||
|
||||
uwsgi_args="--master --pythonpath=lib"
|
||||
|
||||
parse_common_args() {
|
||||
INITIALIZE_TOOL_DEPENDENCIES=1
|
||||
# Pop args meant for common_startup.sh
|
||||
while :
|
||||
do
|
||||
case "$1" in
|
||||
--skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip|--replace-pip)
|
||||
common_startup_args="$common_startup_args $1"
|
||||
shift
|
||||
;;
|
||||
--skip-tool-dependency-initialization)
|
||||
INITIALIZE_TOOL_DEPENDENCIES=0
|
||||
shift
|
||||
;;
|
||||
--skip-venv)
|
||||
skip_venv=1
|
||||
common_startup_args="$common_startup_args $1"
|
||||
shift
|
||||
;;
|
||||
--stop-daemon)
|
||||
common_startup_args="$common_startup_args $1"
|
||||
paster_args="$paster_args $1"
|
||||
uwsgi_args="$uwsgi_args --stop $PID_FILE"
|
||||
stop_daemon_arg_set=1
|
||||
shift
|
||||
;;
|
||||
--restart|restart)
|
||||
if [ "$1" = "--restart" ]
|
||||
then
|
||||
paster_args="$paster_args restart"
|
||||
else
|
||||
paster_args="$paster_args $1"
|
||||
fi
|
||||
uwsgi_args="$uwsgi_args --reload $PID_FILE"
|
||||
daemon_or_restart_arg_set=1
|
||||
shift
|
||||
;;
|
||||
--daemon)
|
||||
paster_args="$paster_args $1"
|
||||
# --daemonize2 waits until after the application has loaded
|
||||
# to daemonize, thus it stops if any errors are found
|
||||
uwsgi_args="$uwsgi_args --daemonize2 $LOG_FILE --safe-pidfile $PID_FILE"
|
||||
daemon_or_restart_arg_set=1
|
||||
shift
|
||||
;;
|
||||
--wait)
|
||||
wait_arg_set=1
|
||||
shift
|
||||
;;
|
||||
"")
|
||||
break
|
||||
;;
|
||||
*)
|
||||
paster_args="$paster_args $1"
|
||||
uwsgi_args="$uwsgi_args $1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
run_common_start_up() {
|
||||
./scripts/common_startup.sh $common_startup_args || exit 1
|
||||
}
|
||||
|
||||
setup_python() {
|
||||
# If there is a .venv/ directory, assume it contains a virtualenv that we
|
||||
# should run this instance in.
|
||||
GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-.venv}"
|
||||
if [ -d "$GALAXY_VIRTUAL_ENV" -a -z "$skip_venv" ];
|
||||
then
|
||||
[ -n "$PYTHONPATH" ] && { echo 'Unsetting $PYTHONPATH'; unset PYTHONPATH; }
|
||||
echo "Activating virtualenv at $GALAXY_VIRTUAL_ENV"
|
||||
. "$GALAXY_VIRTUAL_ENV/bin/activate"
|
||||
fi
|
||||
|
||||
# If you are using --skip-venv we assume you know what you are doing but warn
|
||||
# in case you don't.
|
||||
[ -n "$PYTHONPATH" ] && echo 'WARNING: $PYTHONPATH is set, this can cause problems importing Galaxy dependencies'
|
||||
|
||||
python ./scripts/check_python.py || exit 1
|
||||
}
|
||||
|
||||
find_uwsgi() {
|
||||
# Look for uwsgi
|
||||
if [ -z "$skip_venv" -a -x $GALAXY_VIRTUAL_ENV/bin/uwsgi ]; then
|
||||
UWSGI=$GALAXY_VIRTUAL_ENV/bin/uwsgi
|
||||
elif command -v uwsgi >/dev/null 2>&1; then
|
||||
UWSGI=uwsgi
|
||||
else
|
||||
echo 'ERROR: Could not find uwsgi executable'
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
find_server() {
|
||||
server_config="$1"
|
||||
server_config_style="ini-paste"
|
||||
default_webserver="paste"
|
||||
case "$server_config" in
|
||||
*.y*ml*)
|
||||
server_config_style="yaml"
|
||||
default_webserver="uwsgi" # paste incapable of this
|
||||
;;
|
||||
esac
|
||||
|
||||
APP_WEBSERVER=${APP_WEBSERVER:-$default_webserver}
|
||||
if [ "$APP_WEBSERVER" = "uwsgi" ];
|
||||
then
|
||||
# Look for uwsgi
|
||||
if [ -z "$skip_venv" -a -x $GALAXY_VIRTUAL_ENV/bin/uwsgi ]; then
|
||||
UWSGI=$GALAXY_VIRTUAL_ENV/bin/uwsgi
|
||||
elif command -v uwsgi >/dev/null 2>&1; then
|
||||
UWSGI=uwsgi
|
||||
else
|
||||
echo 'ERROR: Could not find uwsgi executable'
|
||||
exit 1
|
||||
fi
|
||||
run_server="$UWSGI"
|
||||
server_args="--$server_config_style $server_config $uwsgi_args"
|
||||
else
|
||||
run_server="python"
|
||||
server_args="./scripts/paster.py serve $server_config $paster_args --pid-file $PID_FILE --log-file $LOG_FILE $paster_args"
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
"""Entry point for conversion procedure."""
|
||||
found_app_main = False
|
||||
current_section_desc = []
|
||||
current_section_options = []
|
||||
try:
|
||||
sample = sys.argv[1]
|
||||
except KeyError:
|
||||
sample = "config/galaxy.ini.sample"
|
||||
|
||||
for line in open(sample, "r"):
|
||||
is_app_main = line.startswith('[app:main]')
|
||||
if not found_app_main and not is_app_main:
|
||||
continue
|
||||
if is_app_main:
|
||||
found_app_main = True
|
||||
continue
|
||||
|
||||
line = line.strip()
|
||||
if not line.startswith("#"):
|
||||
for section_option in current_section_options:
|
||||
_dump_option(section_option, current_section_desc)
|
||||
current_section_desc = []
|
||||
current_section_options = []
|
||||
if line.startswith("[galaxy_amqp]"):
|
||||
break
|
||||
|
||||
if line.startswith("# ") or "#" == line:
|
||||
current_section_desc.append(line[2:])
|
||||
elif line.startswith("#"):
|
||||
current_section_options.append(line)
|
||||
|
||||
|
||||
def _dump_option(option, current_section_desc):
|
||||
def print_line(line):
|
||||
print((" " * 6) + line)
|
||||
if "=" not in option:
|
||||
print(option)
|
||||
key, default = [s.strip() for s in option.split("=", 1)]
|
||||
key = key[1:] # strip #
|
||||
if default.strip().lower() in ["true", "false"]:
|
||||
default = default.lower()
|
||||
type = "bool"
|
||||
elif default.isdigit():
|
||||
type = "int"
|
||||
else:
|
||||
try:
|
||||
float(default)
|
||||
type = "float"
|
||||
except ValueError:
|
||||
type = "str"
|
||||
if default == "None":
|
||||
default = None
|
||||
print_line("%s:" % key)
|
||||
print_line(" type: %s" % type)
|
||||
if default is not None:
|
||||
print_line(" default: %s" % default)
|
||||
# print_line(" required: false")
|
||||
print_line(" desc: |")
|
||||
for line in current_section_desc:
|
||||
print_line(" %s" % line)
|
||||
print_line("")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -18,7 +18,7 @@ if not os.path.exists( config_file ):
|
||||
sys.exit( 1 )
|
||||
repo = 'lib/tool_shed/galaxy_install/migrate'
|
||||
|
||||
properties = load_app_properties( ini_file=config_file )
|
||||
properties = load_app_properties( config_file=config_file )
|
||||
cp = SafeConfigParser()
|
||||
cp.read( config_file )
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
+144
-142
@@ -87,10 +87,10 @@ class ToolsTestCase( api.ApiTestCase ):
|
||||
return tool_info
|
||||
|
||||
def test_upload1_paste( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
payload = self.dataset_populator.upload_payload( history_id, 'Hello World' )
|
||||
create_response = self._post( "tools", data=payload )
|
||||
self._assert_has_keys( create_response.json(), 'outputs' )
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
payload = self.dataset_populator.upload_payload( history_id, 'Hello World' )
|
||||
create_response = self._post( "tools", data=payload )
|
||||
self._assert_has_keys( create_response.json(), 'outputs' )
|
||||
|
||||
def test_upload_posix_newline_fixes( self ):
|
||||
windows_content = "1\t2\t3\r4\t5\t6\r"
|
||||
@@ -120,174 +120,176 @@ class ToolsTestCase( api.ApiTestCase ):
|
||||
self.assertEquals( rdata_metadata[ "file_ext" ], "rdata" )
|
||||
|
||||
def test_unzip_collection( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
hdca_id = self.__build_pair( history_id, [ "123", "456" ] )
|
||||
inputs = {
|
||||
"input": { "src": "hdca", "id": hdca_id },
|
||||
}
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=True )
|
||||
response = self._run( "__UNZIP_COLLECTION__", history_id, inputs, assert_ok=True )
|
||||
outputs = response[ "outputs" ]
|
||||
self.assertEquals( len(outputs), 2 )
|
||||
output_forward = outputs[ 0 ]
|
||||
output_reverse = outputs[ 1 ]
|
||||
output_forward_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output_forward )
|
||||
output_reverse_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output_reverse )
|
||||
assert output_forward_content.strip() == "123"
|
||||
assert output_reverse_content.strip() == "456"
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
hdca_id = self.__build_pair( history_id, [ "123", "456" ] )
|
||||
inputs = {
|
||||
"input": { "src": "hdca", "id": hdca_id },
|
||||
}
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=True )
|
||||
response = self._run( "__UNZIP_COLLECTION__", history_id, inputs, assert_ok=True )
|
||||
outputs = response[ "outputs" ]
|
||||
self.assertEquals( len(outputs), 2 )
|
||||
output_forward = outputs[ 0 ]
|
||||
output_reverse = outputs[ 1 ]
|
||||
output_forward_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output_forward )
|
||||
output_reverse_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output_reverse )
|
||||
assert output_forward_content.strip() == "123"
|
||||
assert output_reverse_content.strip() == "456"
|
||||
|
||||
output_forward = self.dataset_populator.get_history_dataset_details( history_id, dataset=output_forward )
|
||||
output_reverse = self.dataset_populator.get_history_dataset_details( history_id, dataset=output_reverse )
|
||||
output_forward = self.dataset_populator.get_history_dataset_details( history_id, dataset=output_forward )
|
||||
output_reverse = self.dataset_populator.get_history_dataset_details( history_id, dataset=output_reverse )
|
||||
|
||||
assert output_forward["history_id"] == history_id
|
||||
assert output_reverse["history_id"] == history_id
|
||||
assert output_forward["history_id"] == history_id
|
||||
assert output_reverse["history_id"] == history_id
|
||||
|
||||
def test_unzip_nested( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
hdca_list_id = self.__build_nested_list( history_id )
|
||||
inputs = {
|
||||
"input": {
|
||||
'batch': True,
|
||||
'values': [ { 'src': 'hdca', 'map_over_type': 'paired', 'id': hdca_list_id }],
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
hdca_list_id = self.__build_nested_list( history_id )
|
||||
inputs = {
|
||||
"input": {
|
||||
'batch': True,
|
||||
'values': [ { 'src': 'hdca', 'map_over_type': 'paired', 'id': hdca_list_id }],
|
||||
}
|
||||
}
|
||||
}
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=True )
|
||||
self._run( "__UNZIP_COLLECTION__", history_id, inputs, assert_ok=True )
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=True )
|
||||
self._run( "__UNZIP_COLLECTION__", history_id, inputs, assert_ok=True )
|
||||
|
||||
def test_zip_inputs( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
hda1 = dataset_to_param( self.dataset_populator.new_dataset( history_id, content='1\t2\t3' ) )
|
||||
hda2 = dataset_to_param( self.dataset_populator.new_dataset( history_id, content='4\t5\t6' ) )
|
||||
inputs = {
|
||||
"input_forward": hda1,
|
||||
"input_reverse": hda2,
|
||||
}
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=True )
|
||||
response = self._run( "__ZIP_COLLECTION__", history_id, inputs, assert_ok=True )
|
||||
output_collections = response[ "output_collections" ]
|
||||
self.assertEquals( len(output_collections), 1 )
|
||||
self.dataset_populator.wait_for_job( response["jobs"][0]["id"], assert_ok=True )
|
||||
zipped_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=output_collections[0]["hid"])
|
||||
assert zipped_hdca["collection_type"] == "paired"
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
hda1 = dataset_to_param( self.dataset_populator.new_dataset( history_id, content='1\t2\t3' ) )
|
||||
hda2 = dataset_to_param( self.dataset_populator.new_dataset( history_id, content='4\t5\t6' ) )
|
||||
inputs = {
|
||||
"input_forward": hda1,
|
||||
"input_reverse": hda2,
|
||||
}
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=True )
|
||||
response = self._run( "__ZIP_COLLECTION__", history_id, inputs, assert_ok=True )
|
||||
output_collections = response[ "output_collections" ]
|
||||
self.assertEquals( len(output_collections), 1 )
|
||||
self.dataset_populator.wait_for_job( response["jobs"][0]["id"], assert_ok=True )
|
||||
zipped_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=output_collections[0]["hid"])
|
||||
assert zipped_hdca["collection_type"] == "paired"
|
||||
|
||||
def test_zip_list_inputs( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
hdca1_id = self.dataset_collection_populator.create_list_in_history( history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"] ).json()["id"]
|
||||
hdca2_id = self.dataset_collection_populator.create_list_in_history( history_id, contents=["1\n2\n3\n4", "5\n6\n7\n8"] ).json()["id"]
|
||||
inputs = {
|
||||
"input_forward": { 'batch': True, 'values': [ {"src": "hdca", "id": hdca1_id} ] },
|
||||
"input_reverse": { 'batch': True, 'values': [ {"src": "hdca", "id": hdca2_id} ] },
|
||||
}
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=True )
|
||||
response = self._run( "__ZIP_COLLECTION__", history_id, inputs, assert_ok=True )
|
||||
implicit_collections = response[ "implicit_collections" ]
|
||||
self.assertEquals( len(implicit_collections), 1 )
|
||||
self.dataset_populator.wait_for_job( response["jobs"][0]["id"], assert_ok=True )
|
||||
zipped_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=implicit_collections[0]["hid"])
|
||||
assert zipped_hdca["collection_type"] == "list:paired"
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
hdca1_id = self.dataset_collection_populator.create_list_in_history( history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"] ).json()["id"]
|
||||
hdca2_id = self.dataset_collection_populator.create_list_in_history( history_id, contents=["1\n2\n3\n4", "5\n6\n7\n8"] ).json()["id"]
|
||||
inputs = {
|
||||
"input_forward": { 'batch': True, 'values': [ {"src": "hdca", "id": hdca1_id} ] },
|
||||
"input_reverse": { 'batch': True, 'values': [ {"src": "hdca", "id": hdca2_id} ] },
|
||||
}
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=True )
|
||||
response = self._run( "__ZIP_COLLECTION__", history_id, inputs, assert_ok=True )
|
||||
implicit_collections = response[ "implicit_collections" ]
|
||||
self.assertEquals( len(implicit_collections), 1 )
|
||||
self.dataset_populator.wait_for_job( response["jobs"][0]["id"], assert_ok=True )
|
||||
zipped_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=implicit_collections[0]["hid"])
|
||||
assert zipped_hdca["collection_type"] == "list:paired"
|
||||
|
||||
def test_filter_failed( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
ok_hdca_id = self.dataset_collection_populator.create_list_in_history( history_id, contents=["0", "1", "0", "1"] ).json()["id"]
|
||||
exit_code_inputs = {
|
||||
"input": { 'batch': True, 'values': [ {"src": "hdca", "id": ok_hdca_id} ] },
|
||||
}
|
||||
response = self._run( "exit_code_from_file", history_id, exit_code_inputs, assert_ok=False ).json()
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=False )
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
history_id = self.dataset_populator.new_history()
|
||||
ok_hdca_id = self.dataset_collection_populator.create_list_in_history( history_id, contents=["0", "1", "0", "1"] ).json()["id"]
|
||||
exit_code_inputs = {
|
||||
"input": { 'batch': True, 'values': [ {"src": "hdca", "id": ok_hdca_id} ] },
|
||||
}
|
||||
response = self._run( "exit_code_from_file", history_id, exit_code_inputs, assert_ok=False ).json()
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=False )
|
||||
|
||||
mixed_implicit_collections = response[ "implicit_collections" ]
|
||||
self.assertEquals( len(mixed_implicit_collections), 1 )
|
||||
mixed_hdca_hid = mixed_implicit_collections[0]["hid"]
|
||||
mixed_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=mixed_hdca_hid, wait=False)
|
||||
mixed_implicit_collections = response[ "implicit_collections" ]
|
||||
self.assertEquals( len(mixed_implicit_collections), 1 )
|
||||
mixed_hdca_hid = mixed_implicit_collections[0]["hid"]
|
||||
mixed_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=mixed_hdca_hid, wait=False)
|
||||
|
||||
def get_state(dce):
|
||||
return dce["object"]["state"]
|
||||
def get_state(dce):
|
||||
return dce["object"]["state"]
|
||||
|
||||
mixed_states = [get_state(_) for _ in mixed_hdca["elements"]]
|
||||
assert mixed_states == [u"ok", u"error", u"ok", u"error"], mixed_states
|
||||
inputs = {
|
||||
"input": { "src": "hdca", "id": mixed_hdca["id"] },
|
||||
}
|
||||
response = self._run( "__FILTER_FAILED_DATASETS__", history_id, inputs, assert_ok=False ).json()
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=False )
|
||||
filter_output_collections = response[ "output_collections" ]
|
||||
self.assertEquals( len(filter_output_collections), 1 )
|
||||
filtered_hid = filter_output_collections[0]["hid"]
|
||||
filtered_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=filtered_hid, wait=False)
|
||||
filtered_states = [get_state(_) for _ in filtered_hdca["elements"]]
|
||||
assert filtered_states == [u"ok", u"ok"], filtered_states
|
||||
mixed_states = [get_state(_) for _ in mixed_hdca["elements"]]
|
||||
assert mixed_states == [u"ok", u"error", u"ok", u"error"], mixed_states
|
||||
inputs = {
|
||||
"input": { "src": "hdca", "id": mixed_hdca["id"] },
|
||||
}
|
||||
response = self._run( "__FILTER_FAILED_DATASETS__", history_id, inputs, assert_ok=False ).json()
|
||||
self.dataset_populator.wait_for_history( history_id, assert_ok=False )
|
||||
filter_output_collections = response[ "output_collections" ]
|
||||
self.assertEquals( len(filter_output_collections), 1 )
|
||||
filtered_hid = filter_output_collections[0]["hid"]
|
||||
filtered_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=filtered_hid, wait=False)
|
||||
filtered_states = [get_state(_) for _ in filtered_hdca["elements"]]
|
||||
assert filtered_states == [u"ok", u"ok"], filtered_states
|
||||
|
||||
@skip_without_tool( "multi_select" )
|
||||
def test_multi_select_as_list( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
inputs = {
|
||||
"select_ex": ["--ex1", "ex2"],
|
||||
}
|
||||
response = self._run( "multi_select", history_id, inputs, assert_ok=True )
|
||||
output = response[ "outputs" ][ 0 ]
|
||||
output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output )
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
inputs = {
|
||||
"select_ex": ["--ex1", "ex2"],
|
||||
}
|
||||
response = self._run( "multi_select", history_id, inputs, assert_ok=True )
|
||||
output = response[ "outputs" ][ 0 ]
|
||||
output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output )
|
||||
|
||||
assert output1_content == "--ex1,ex2"
|
||||
assert output1_content == "--ex1,ex2"
|
||||
|
||||
@skip_without_tool( "multi_select" )
|
||||
def test_multi_select_optional( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
inputs = {
|
||||
"select_ex": ["--ex1"],
|
||||
"select_optional": None,
|
||||
}
|
||||
response = self._run( "multi_select", history_id, inputs, assert_ok=True )
|
||||
output = response[ "outputs" ]
|
||||
output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output[ 0 ] )
|
||||
output2_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output[ 1 ] )
|
||||
assert output1_content.strip() == "--ex1"
|
||||
assert output2_content.strip() == "None", output2_content
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
inputs = {
|
||||
"select_ex": ["--ex1"],
|
||||
"select_optional": None,
|
||||
}
|
||||
response = self._run( "multi_select", history_id, inputs, assert_ok=True )
|
||||
output = response[ "outputs" ]
|
||||
output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output[ 0 ] )
|
||||
output2_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output[ 1 ] )
|
||||
assert output1_content.strip() == "--ex1"
|
||||
assert output2_content.strip() == "None", output2_content
|
||||
|
||||
@skip_without_tool( "library_data" )
|
||||
def test_library_data_param( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
ld = LibraryPopulator( self ).new_library_dataset( "lda_test_library" )
|
||||
inputs = {
|
||||
"library_dataset": ld[ "ldda_id" ],
|
||||
"library_dataset_multiple": [ld[ "ldda_id" ], ld[ "ldda_id" ]]
|
||||
}
|
||||
response = self._run( "library_data", history_id, inputs, assert_ok=True )
|
||||
output = response[ "outputs" ]
|
||||
output_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output[ 0 ] )
|
||||
assert output_content == "TestData\n", output_content
|
||||
output_multiple_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output[ 1 ] )
|
||||
assert output_multiple_content == "TestData\nTestData\n", output_multiple_content
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
ld = LibraryPopulator( self ).new_library_dataset( "lda_test_library" )
|
||||
inputs = {
|
||||
"library_dataset": ld[ "ldda_id" ],
|
||||
"library_dataset_multiple": [ld[ "ldda_id" ], ld[ "ldda_id" ]]
|
||||
}
|
||||
response = self._run( "library_data", history_id, inputs, assert_ok=True )
|
||||
output = response[ "outputs" ]
|
||||
output_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output[ 0 ] )
|
||||
assert output_content == "TestData\n", output_content
|
||||
output_multiple_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output[ 1 ] )
|
||||
assert output_multiple_content == "TestData\nTestData\n", output_multiple_content
|
||||
|
||||
@skip_without_tool( "multi_data_param" )
|
||||
def test_multidata_param( self ):
|
||||
history_id = self.dataset_populator.new_history()
|
||||
hda1 = dataset_to_param( self.dataset_populator.new_dataset( history_id, content='1\t2\t3' ) )
|
||||
hda2 = dataset_to_param( self.dataset_populator.new_dataset( history_id, content='4\t5\t6' ) )
|
||||
inputs = {
|
||||
"f1": { 'batch': False, 'values': [ hda1, hda2 ] },
|
||||
"f2": { 'batch': False, 'values': [ hda2, hda1 ] },
|
||||
}
|
||||
response = self._run( "multi_data_param", history_id, inputs, assert_ok=True )
|
||||
output1 = response[ "outputs" ][ 0 ]
|
||||
output2 = response[ "outputs" ][ 1 ]
|
||||
output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 )
|
||||
output2_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output2 )
|
||||
assert output1_content == "1\t2\t3\n4\t5\t6\n", output1_content
|
||||
assert output2_content == "4\t5\t6\n1\t2\t3\n", output2_content
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
hda1 = dataset_to_param( self.dataset_populator.new_dataset( history_id, content='1\t2\t3' ) )
|
||||
hda2 = dataset_to_param( self.dataset_populator.new_dataset( history_id, content='4\t5\t6' ) )
|
||||
inputs = {
|
||||
"f1": { 'batch': False, 'values': [ hda1, hda2 ] },
|
||||
"f2": { 'batch': False, 'values': [ hda2, hda1 ] },
|
||||
}
|
||||
response = self._run( "multi_data_param", history_id, inputs, assert_ok=True )
|
||||
output1 = response[ "outputs" ][ 0 ]
|
||||
output2 = response[ "outputs" ][ 1 ]
|
||||
output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 )
|
||||
output2_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output2 )
|
||||
assert output1_content == "1\t2\t3\n4\t5\t6\n", output1_content
|
||||
assert output2_content == "4\t5\t6\n1\t2\t3\n", output2_content
|
||||
|
||||
@skip_without_tool( "cat1" )
|
||||
def test_run_cat1( self ):
|
||||
# Run simple non-upload tool with an input data parameter.
|
||||
history_id = self.dataset_populator.new_history()
|
||||
new_dataset = self.dataset_populator.new_dataset( history_id, content='Cat1Test' )
|
||||
inputs = dict(
|
||||
input1=dataset_to_param( new_dataset ),
|
||||
)
|
||||
outputs = self._cat1_outputs( history_id, inputs=inputs )
|
||||
self.assertEquals( len( outputs ), 1 )
|
||||
output1 = outputs[ 0 ]
|
||||
output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 )
|
||||
self.assertEqual( output1_content.strip(), "Cat1Test" )
|
||||
with self.dataset_populator.test_history() as history_id:
|
||||
# Run simple non-upload tool with an input data parameter.
|
||||
history_id = self.dataset_populator.new_history()
|
||||
new_dataset = self.dataset_populator.new_dataset( history_id, content='Cat1Test' )
|
||||
inputs = dict(
|
||||
input1=dataset_to_param( new_dataset ),
|
||||
)
|
||||
outputs = self._cat1_outputs( history_id, inputs=inputs )
|
||||
self.assertEquals( len( outputs ), 1 )
|
||||
output1 = outputs[ 0 ]
|
||||
output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 )
|
||||
self.assertEqual( output1_content.strip(), "Cat1Test" )
|
||||
|
||||
@skip_without_tool( "cat1" )
|
||||
def test_run_cat1_listified_param( self ):
|
||||
|
||||
@@ -330,12 +330,18 @@ class WorkflowsApiTestCase( BaseWorkflowsApiTestCase ):
|
||||
def test_upload_deprecated( self ):
|
||||
self.__test_upload( use_deprecated_route=True )
|
||||
|
||||
def __test_upload( self, use_deprecated_route=False, name="test_import", workflow=None, assert_ok=True ):
|
||||
def test_import_tools_requires_admin( self ):
|
||||
response = self.__test_upload( import_tools=True, assert_ok=False )
|
||||
assert response.status_code == 403
|
||||
|
||||
def __test_upload( self, use_deprecated_route=False, name="test_import", workflow=None, assert_ok=True, import_tools=False ):
|
||||
if workflow is None:
|
||||
workflow = self.workflow_populator.load_workflow( name=name )
|
||||
data = dict(
|
||||
workflow=dumps( workflow ),
|
||||
)
|
||||
if import_tools:
|
||||
data["import_tools"] = import_tools
|
||||
if use_deprecated_route:
|
||||
route = "workflows/upload"
|
||||
else:
|
||||
|
||||
+4
-1
@@ -13,6 +13,7 @@ from .interactor import GalaxyInteractorApi as BaseInteractor
|
||||
from .twilltestcase import FunctionalTestCase
|
||||
|
||||
TEST_USER = "user@bx.psu.edu"
|
||||
ADMIN_TEST_USER = "test@bx.psu.edu"
|
||||
DEFAULT_OTHER_USER = "otheruser@bx.psu.edu" # A second user for API testing.
|
||||
|
||||
|
||||
@@ -107,7 +108,9 @@ class ApiTestInteractor( BaseInteractor ):
|
||||
"""
|
||||
|
||||
def __init__( self, test_case ):
|
||||
super( ApiTestInteractor, self ).__init__( test_case, test_user=TEST_USER )
|
||||
admin = getattr( test_case, "require_admin_user", False )
|
||||
test_user = TEST_USER if not admin else ADMIN_TEST_USER
|
||||
super( ApiTestInteractor, self ).__init__( test_case, test_user=test_user )
|
||||
|
||||
# This variant the lower level get and post methods are meant to be used
|
||||
# directly to test API - instead of relying on higher-level constructs for
|
||||
|
||||
+19
-14
@@ -327,7 +327,7 @@ class GalaxyInteractorApi( object ):
|
||||
return self._state_ready( state, error_msg="Job in error state." )
|
||||
except Exception:
|
||||
if VERBOSE_ERRORS:
|
||||
self._summarize_history_errors( history_id )
|
||||
self._summarize_history( history_id )
|
||||
raise
|
||||
|
||||
def __history_ready( self, history_id ):
|
||||
@@ -339,49 +339,54 @@ class GalaxyInteractorApi( object ):
|
||||
return self._state_ready( state, error_msg="History in error state." )
|
||||
except Exception:
|
||||
if VERBOSE_ERRORS:
|
||||
self._summarize_history_errors( history_id )
|
||||
self._summarize_history( history_id )
|
||||
raise
|
||||
|
||||
def _summarize_history_errors( self, history_id ):
|
||||
def _summarize_history( self, history_id ):
|
||||
if history_id is None:
|
||||
raise ValueError("_summarize_history_errors passed empty history_id")
|
||||
print("History with id %s in error - summary of datasets in error below." % history_id)
|
||||
raise ValueError("_summarize_history passed empty history_id")
|
||||
print("Problem in history with id %s - summary of datasets below." % history_id)
|
||||
try:
|
||||
history_contents = self.__contents( history_id )
|
||||
except Exception:
|
||||
print("*TEST FRAMEWORK FAILED TO FETCH HISTORY DETAILS*")
|
||||
|
||||
for history_content in history_contents:
|
||||
if history_content[ 'history_content_type'] != 'dataset':
|
||||
continue
|
||||
|
||||
dataset = history_content
|
||||
if dataset[ 'state' ] != 'error':
|
||||
continue
|
||||
|
||||
print(ERROR_MESSAGE_DATASET_SEP)
|
||||
dataset_id = dataset.get( 'id', None )
|
||||
print("| %d - %s (HID - NAME) " % ( int( dataset['hid'] ), dataset['name'] ))
|
||||
if history_content[ 'history_content_type'] == 'dataset_collection':
|
||||
history_contents_json = self._get( "histories/%s/contents/dataset_collections/%s" % (history_id, history_content["id"] ) ).json()
|
||||
print("| Dataset Collection: %s" % history_contents_json)
|
||||
continue
|
||||
|
||||
try:
|
||||
dataset_info = self._dataset_info( history_id, dataset_id )
|
||||
print("| Dataset State:")
|
||||
print(self.format_for_summary(dataset_info.get("state"), "Dataset state is unknown."))
|
||||
print("| Dataset Blurb:")
|
||||
print(self.format_for_error( dataset_info.get( "misc_blurb", "" ), "Dataset blurb was empty." ))
|
||||
print(self.format_for_summary( dataset_info.get( "misc_blurb", "" ), "Dataset blurb was empty." ))
|
||||
print("| Dataset Info:")
|
||||
print(self.format_for_error( dataset_info.get( "misc_info", "" ), "Dataset info is empty." ))
|
||||
print(self.format_for_summary( dataset_info.get( "misc_info", "" ), "Dataset info is empty." ))
|
||||
print("| Peek:")
|
||||
print(self.format_for_summary( dataset_info.get( "peek", ""), "Peek unavilable."))
|
||||
except Exception:
|
||||
print("| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*")
|
||||
try:
|
||||
provenance_info = self._dataset_provenance( history_id, dataset_id )
|
||||
print("| Dataset Job Standard Output:")
|
||||
print(self.format_for_error( provenance_info.get( "stdout", "" ), "Standard output was empty." ))
|
||||
print(self.format_for_summary( provenance_info.get( "stdout", "" ), "Standard output was empty." ))
|
||||
print("| Dataset Job Standard Error:")
|
||||
print(self.format_for_error( provenance_info.get( "stderr", "" ), "Standard error was empty." ))
|
||||
print(self.format_for_summary( provenance_info.get( "stderr", "" ), "Standard error was empty." ))
|
||||
except Exception:
|
||||
print("| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*")
|
||||
print("|")
|
||||
print(ERROR_MESSAGE_DATASET_SEP)
|
||||
|
||||
def format_for_error( self, blob, empty_message, prefix="| " ):
|
||||
def format_for_summary( self, blob, empty_message, prefix="| " ):
|
||||
contents = "\n".join([ "%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r") ] )
|
||||
return contents or "%s*%s*" % ( prefix, empty_message )
|
||||
|
||||
|
||||
+24
-7
@@ -1,3 +1,4 @@
|
||||
import contextlib
|
||||
import json
|
||||
import time
|
||||
|
||||
@@ -80,18 +81,20 @@ class BaseDatasetPopulator( object ):
|
||||
|
||||
def new_dataset( self, history_id, content='TestData123', wait=False, **kwds ):
|
||||
payload = self.upload_payload( history_id, content, **kwds )
|
||||
run_response = self._post( "tools", data=payload ).json()
|
||||
run_response = self._post( "tools", data=payload )
|
||||
run = run_response.json()
|
||||
if wait:
|
||||
job = run_response["jobs"][0]
|
||||
assert run_response.status_code == 200, run
|
||||
job = run["jobs"][0]
|
||||
self.wait_for_job(job["id"])
|
||||
self.wait_for_history(history_id, assert_ok=True)
|
||||
return run_response["outputs"][0]
|
||||
return run["outputs"][0]
|
||||
|
||||
def wait_for_history( self, history_id, assert_ok=False, timeout=DEFAULT_TIMEOUT ):
|
||||
try:
|
||||
return wait_on_state( lambda: self._get( "histories/%s" % history_id ), assert_ok=assert_ok, timeout=timeout )
|
||||
except AssertionError:
|
||||
self._summarize_history_errors( history_id )
|
||||
self._summarize_history( history_id )
|
||||
raise
|
||||
|
||||
def wait_for_job( self, job_id, assert_ok=False, timeout=DEFAULT_TIMEOUT ):
|
||||
@@ -100,9 +103,21 @@ class BaseDatasetPopulator( object ):
|
||||
def get_job_details( self, job_id, full=False ):
|
||||
return self._get( "jobs/%s?full=%s" % (job_id, full) )
|
||||
|
||||
def _summarize_history_errors( self, history_id ):
|
||||
def _summarize_history( self, history_id ):
|
||||
pass
|
||||
|
||||
@contextlib.contextmanager
|
||||
def test_history(self, **kwds):
|
||||
# TODO: In the future allow targetting a specfic history here
|
||||
# and/or deleting everything in the resulting history when done.
|
||||
# These would be cool options for remote Galaxy test execution.
|
||||
try:
|
||||
history_id = self.new_history()
|
||||
yield history_id
|
||||
except Exception:
|
||||
self._summarize_history(history_id)
|
||||
raise
|
||||
|
||||
def new_history( self, **kwds ):
|
||||
name = kwds.get( "name", "API Test History" )
|
||||
create_history_response = self._post( "histories", data=dict( name=name ) )
|
||||
@@ -127,6 +142,8 @@ class BaseDatasetPopulator( object ):
|
||||
upload_params[ "files_0|to_posix_lines"] = kwds[ "to_posix_lines" ]
|
||||
if "space_to_tab" in kwds:
|
||||
upload_params[ "files_0|space_to_tab" ] = kwds[ "space_to_tab" ]
|
||||
if "auto_decompress" in kwds:
|
||||
upload_params[ "files_0|auto_decompress" ] = kwds[ "auto_decompress" ]
|
||||
return self.run_tool_payload(
|
||||
tool_id='upload1',
|
||||
inputs=upload_params,
|
||||
@@ -217,8 +234,8 @@ class DatasetPopulator( BaseDatasetPopulator ):
|
||||
def _get( self, route ):
|
||||
return self.galaxy_interactor.get( route )
|
||||
|
||||
def _summarize_history_errors( self, history_id ):
|
||||
self.galaxy_interactor._summarize_history_errors( history_id )
|
||||
def _summarize_history( self, history_id ):
|
||||
self.galaxy_interactor._summarize_history( history_id )
|
||||
|
||||
def wait_for_dataset(self, history_id, dataset_id, assert_ok=False, timeout=DEFAULT_TIMEOUT):
|
||||
return wait_on_state(lambda: self._get("histories/%s/contents/%s" % (history_id, dataset_id)), assert_ok=assert_ok, timeout=timeout)
|
||||
|
||||
@@ -24,5 +24,6 @@
|
||||
<datatype extension="vcf" type="galaxy.datatypes.tabular:Vcf" display_in_upload="true"/>
|
||||
<datatype extension="bgzip" type="galaxy.datatypes.binary:Binary" subclass="true" />
|
||||
<datatype extension="vcf_bgzip" type="galaxy.datatypes.tabular:VcfGz" type_extension="bgzip" subclass="true" display_in_upload="true"/>
|
||||
<datatype extension="html" type="galaxy.datatypes.text:Html" mimetype="text/html"/>
|
||||
</registration>
|
||||
</datatypes>
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Integration tests various upload aspects.
|
||||
|
||||
This file checks upload options that require different non-default Galaxy
|
||||
configuration options. More vanilla upload options and behaviors are tested
|
||||
with the API test framework (located in test_tools.py).
|
||||
|
||||
These options include:
|
||||
- The config optiond check_upload_content and allow_path_paste.
|
||||
- The upload API parameter auto_decompress.
|
||||
- Checking for malicious content in uploads of compressed files.
|
||||
- Restricting file:// uploads to admins and allowing them only when
|
||||
allow_path_paste is set to True.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from base import integration_util
|
||||
from base.populators import DatasetPopulator
|
||||
|
||||
SCRIPT_DIR = os.path.normpath(os.path.dirname(__file__))
|
||||
TEST_DATA_DIRECTORY = os.path.join(SCRIPT_DIR, os.pardir, os.pardir, "test-data")
|
||||
|
||||
|
||||
class BaseCheckUploadContentConfigurationTestCase(integration_util.IntegrationTestCase):
|
||||
|
||||
framework_tool_and_types = True
|
||||
|
||||
def setUp(self):
|
||||
super(BaseCheckUploadContentConfigurationTestCase, self).setUp()
|
||||
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
|
||||
self.history_id = self.dataset_populator.new_history()
|
||||
|
||||
|
||||
class NonAdminsCannotPasteFilePathTestCase(BaseCheckUploadContentConfigurationTestCase):
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
config["allow_path_paste"] = True
|
||||
|
||||
def test(self):
|
||||
payload = self.dataset_populator.upload_payload(
|
||||
self.history_id, 'file://%s/1.RData' % TEST_DATA_DIRECTORY, ext="binary"
|
||||
)
|
||||
create_response = self._post( "tools", data=payload )
|
||||
# Ideally this would be 403 but the tool API endpoint isn't using
|
||||
# the newer API decorator that handles those details.
|
||||
assert create_response.status_code >= 400
|
||||
|
||||
|
||||
class AdminsCanPasteFilePathsTestCase(BaseCheckUploadContentConfigurationTestCase):
|
||||
|
||||
require_admin_user = True
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
config["allow_path_paste"] = True
|
||||
|
||||
def test(self):
|
||||
payload = self.dataset_populator.upload_payload(
|
||||
self.history_id, 'file://%s/random-file' % TEST_DATA_DIRECTORY,
|
||||
)
|
||||
create_response = self._post( "tools", data=payload )
|
||||
# Ideally this would be 403 but the tool API endpoint isn't using
|
||||
# the newer API decorator that handles those details.
|
||||
assert create_response.status_code == 200
|
||||
|
||||
|
||||
class DefaultBinaryContentFiltersTestCase(BaseCheckUploadContentConfigurationTestCase):
|
||||
|
||||
require_admin_user = True
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
config["allow_path_paste"] = True
|
||||
|
||||
def test_random_binary_allowed(self):
|
||||
dataset = self.dataset_populator.new_dataset(
|
||||
self.history_id, 'file://%s/random-file' % TEST_DATA_DIRECTORY, file_type="auto", wait=True
|
||||
)
|
||||
dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset)
|
||||
assert dataset["file_ext"] == "data", dataset
|
||||
|
||||
def test_gzipped_html_content_blocked_by_default(self):
|
||||
dataset = self.dataset_populator.new_dataset(
|
||||
self.history_id, 'file://%s/bad.html.gz' % TEST_DATA_DIRECTORY, file_type="auto", wait=True
|
||||
)
|
||||
dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset)
|
||||
assert dataset["file_size"] == 0
|
||||
|
||||
|
||||
class DisableContentCheckingTestCase(BaseCheckUploadContentConfigurationTestCase):
|
||||
|
||||
require_admin_user = True
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
config["allow_path_paste"] = True
|
||||
config["check_upload_content"] = False
|
||||
|
||||
def test_gzipped_html_content_now_allowed(self):
|
||||
dataset = self.dataset_populator.new_dataset(
|
||||
self.history_id, 'file://%s/bad.html.gz' % TEST_DATA_DIRECTORY, file_type="auto", wait=True
|
||||
)
|
||||
dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset)
|
||||
# Same file was empty above!
|
||||
assert dataset["file_size"] != 0
|
||||
|
||||
|
||||
class AutoDecompressTestCase(BaseCheckUploadContentConfigurationTestCase):
|
||||
|
||||
require_admin_user = True
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
config["allow_path_paste"] = True
|
||||
|
||||
def test_auto_decompress_off(self):
|
||||
dataset = self.dataset_populator.new_dataset(
|
||||
self.history_id, 'file://%s/1.sam.gz' % TEST_DATA_DIRECTORY, file_type="auto", auto_decompress=False, wait=True
|
||||
)
|
||||
dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset)
|
||||
assert dataset["file_ext"] == "data", dataset
|
||||
|
||||
def test_auto_decompress_on(self):
|
||||
dataset = self.dataset_populator.new_dataset(
|
||||
self.history_id, 'file://%s/1.sam.gz' % TEST_DATA_DIRECTORY, file_type="auto", wait=True
|
||||
)
|
||||
dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset)
|
||||
assert dataset["file_ext"] == "sam", dataset
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,188 @@
|
||||
# ---- HTTP Server ----------------------------------------------------------
|
||||
|
||||
[server:main]
|
||||
|
||||
use = egg:Paste#http
|
||||
port = 9009
|
||||
|
||||
# The address on which to listen. By default, only listen to localhost
|
||||
# (the Tool Shed will not be accessible over the network).
|
||||
# Use '0.0.0.0' to listen on all available network interfaces.
|
||||
#host = 0.0.0.0
|
||||
host = 127.0.0.1
|
||||
|
||||
use_threadpool = true
|
||||
threadpool_workers = 10
|
||||
# Set the number of seconds a thread can work before you should kill it
|
||||
# (assuming it will never finish) to 3 hours.
|
||||
threadpool_kill_thread_limit = 10800
|
||||
|
||||
# Define the proxy-prefix filter.
|
||||
[filter:proxy-prefix]
|
||||
use = egg:PasteDeploy#prefix
|
||||
prefix = /shed
|
||||
|
||||
# ---- Galaxy Tool Shed -----------------------------------------------------
|
||||
|
||||
[app:main]
|
||||
|
||||
filter-with = proxy-prefix
|
||||
|
||||
# Specifies the factory for the universe WSGI application
|
||||
paste.app_factory = galaxy.webapps.tool_shed.buildapp:app_factory
|
||||
|
||||
# Verbosity of console log messages. Acceptable values can be found here:
|
||||
# https://docs.python.org/2/library/logging.html#logging-levels
|
||||
#log_level = DEBUG
|
||||
|
||||
# By default, the Tool Shed uses a SQLite database at 'database/community.sqlite'. You
|
||||
# may use a SQLAlchemy connection string to specify an external database
|
||||
# instead. This string takes many options which are explained in detail in the
|
||||
# config file documentation.
|
||||
#database_connection = sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE
|
||||
|
||||
# Where the hgweb.config file is stored.
|
||||
# The default is the Galaxy installation directory.
|
||||
#hgweb_config_dir = None
|
||||
|
||||
# Where tool shed repositories are stored.
|
||||
#file_path = database/community_files
|
||||
|
||||
# Temporary storage for additional datasets,
|
||||
# this should be shared through the cluster
|
||||
#new_file_path = database/tmp
|
||||
|
||||
# File containing old-style genome builds
|
||||
#builds_file_path = tool-data/shared/ucsc/builds.txt
|
||||
|
||||
# Format string used when showing date and time information.
|
||||
# The string may contain:
|
||||
# - the directives used by Python time.strftime() function (see
|
||||
# https://docs.python.org/2/library/time.html#time.strftime ),
|
||||
# - $locale (complete format string for the server locale),
|
||||
# - $iso8601 (complete format string as specified by ISO 8601 international
|
||||
# standard).
|
||||
#pretty_datetime_format = $locale (UTC)
|
||||
|
||||
# -- Repository and Tool search
|
||||
# Using the script located at scripts/build_ts_whoosh_index.py
|
||||
# you can generate search index and allow full text API searching over
|
||||
# the repositories and tools within the Tool Shed given that you specify
|
||||
# the following two config options.
|
||||
#toolshed_search_on = True
|
||||
#whoosh_index_dir = database/toolshed_whoosh_indexes
|
||||
|
||||
# The following boosts are used to customize this instance's TS search.
|
||||
# The higher the boost, the more importance the scoring algorithm gives to the
|
||||
# given field.
|
||||
|
||||
# For searching repositories at /api/repositories:
|
||||
#repo_name_boost = 0.9
|
||||
#repo_description_boost = 0.6
|
||||
#repo_long_description_boost = 0.5
|
||||
#repo_homepage_url_boost = 0.3
|
||||
#repo_remote_repository_url_boost = 0.2
|
||||
#repo_owner_username_boost = 0.3
|
||||
|
||||
# For searching tools at /api/tools
|
||||
#tool_name_boost = 1.2
|
||||
#tool_description_boost = 0.6
|
||||
#tool_help_boost = 0.4
|
||||
#tool_repo_owner_username = 0.3
|
||||
|
||||
# -- Analytics
|
||||
|
||||
# You can enter tracking code here to track visitor's behavior
|
||||
# through your Google Analytics account. Example: UA-XXXXXXXX-Y
|
||||
#ga_code = None
|
||||
|
||||
# -- Users and Security
|
||||
|
||||
# The Tool Shed encodes various internal values when these values will be output in
|
||||
# some format (for example, in a URL or cookie). You should set a key to be
|
||||
# used by the algorithm that encodes and decodes these values. It can be any
|
||||
# string. If left unchanged, anyone could construct a cookie that would grant
|
||||
# them access to others' sessions.
|
||||
# One simple way to generate a value for this is with the shell command:
|
||||
# python -c 'import time; print time.time()' | md5sum | cut -f 1 -d ' '
|
||||
#id_secret = changethisinproductiontoo
|
||||
|
||||
# User authentication can be delegated to an upstream proxy server (usually
|
||||
# Apache). The upstream proxy should set a REMOTE_USER header in the request.
|
||||
# Enabling remote user disables regular logins. For more information, see:
|
||||
# https://wiki.galaxyproject.org/Admin/Config/ApacheProxy
|
||||
#use_remote_user = False
|
||||
|
||||
# If use_remote_user is enabled, anyone who can log in to the Galaxy host may
|
||||
# impersonate any other user by simply sending the appropriate header. Thus a
|
||||
# secret shared between the upstream proxy server, and Galaxy is required.
|
||||
# If anyone other than the Galaxy user is using the server, then apache/nginx
|
||||
# should pass a value in the header 'GX_SECRET' that is identical the one below
|
||||
#remote_user_secret = changethisinproductiontoo
|
||||
|
||||
# Configuration for debugging middleware
|
||||
#debug = False
|
||||
|
||||
# Check for WSGI compliance.
|
||||
#use_lint = False
|
||||
|
||||
# Intercept print statements and show them on the returned page.
|
||||
#use_printdebug = True
|
||||
|
||||
# NEVER enable this on a public site (even test or QA)
|
||||
#use_interactive = true
|
||||
|
||||
# Administrative users - set this to a comma-separated list of valid Tool Shed
|
||||
# users (email addresses). These users will have access to the Admin section
|
||||
# of the server, and will have access to create users, groups, roles,
|
||||
# libraries, and more.
|
||||
#admin_users = None
|
||||
|
||||
# Force everyone to log in (disable anonymous access)
|
||||
#require_login = False
|
||||
|
||||
# For use by email messages sent from the tool shed
|
||||
#smtp_server = smtp.your_tool_shed_server
|
||||
#email_from = your_tool_shed_email@server
|
||||
|
||||
# If your SMTP server requires a username and password, you can provide them
|
||||
# here (password in cleartext here, but if your server supports STARTTLS it
|
||||
# will be sent over the network encrypted).
|
||||
#smtp_username = None
|
||||
#smtp_password = None
|
||||
|
||||
# If your SMTP server requires SSL from the beginning of the connection
|
||||
#smtp_ssl = False
|
||||
|
||||
# The URL linked by the "Support" link in the "Help" menu.
|
||||
#support_url = https://wiki.galaxyproject.org/Support
|
||||
|
||||
# Address to join mailing list
|
||||
#mailing_join_addr = galaxy-announce-join@bx.psu.edu
|
||||
|
||||
# Write thread status periodically to 'heartbeat.log' (careful, uses disk
|
||||
# space rapidly!)
|
||||
#use_heartbeat = True
|
||||
|
||||
# Profiling middleware (cProfile based)
|
||||
#use_profile = True
|
||||
|
||||
# Enable creation of Galaxy flavor Docker Image
|
||||
#enable_galaxy_flavor_docker_image = False
|
||||
|
||||
# Show a message box under the masthead.
|
||||
#message_box_visible = False
|
||||
#message_box_content = None
|
||||
#message_box_class = info
|
||||
|
||||
# Serving static files (needed if running standalone)
|
||||
#static_enabled = True
|
||||
#static_cache_time = 360
|
||||
#static_dir = static/
|
||||
#static_images_dir = static/images
|
||||
#static_favicon_dir = static/favicon.ico
|
||||
#static_scripts_dir = static/scripts/
|
||||
#static_style_dir = static/style/blue
|
||||
|
||||
# Sentry (getsentry.com) DSN for catching bugs.
|
||||
#sentry_dsn = None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -39,16 +39,12 @@ prefix = /reports
|
||||
|
||||
# Specifies the factory for the universe WSGI application
|
||||
paste.app_factory = galaxy.webapps.reports.buildapp:app_factory
|
||||
|
||||
# Verbosity of console log messages. Acceptable values can be found here:
|
||||
# https://docs.python.org/2/library/logging.html#logging-levels
|
||||
#log_level = DEBUG
|
||||
log_level = DEBUG
|
||||
|
||||
# Database connection
|
||||
# Galaxy reports are intended for production Galaxy instances, so sqlite (and the default value
|
||||
# below) is not supported. An SQLAlchemy connection string should be used specify an external
|
||||
# database.
|
||||
#database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
|
||||
# Galaxy reports are intended for production Galaxy instances, so sqlite is not supported.
|
||||
# You may use a SQLAlchemy connection string to specify an external database.
|
||||
#database_connection = postgres:///galaxy_test?user=postgres&password=postgres
|
||||
|
||||
# Where dataset files are saved
|
||||
#file_path = database/files
|
||||
@@ -59,6 +55,13 @@ paste.app_factory = galaxy.webapps.reports.buildapp:app_factory
|
||||
# used for the cache
|
||||
#template_cache_path = database/compiled_templates/reports
|
||||
|
||||
# Session support (beaker)
|
||||
use_beaker_session = True
|
||||
session_type = memory
|
||||
session_data_dir = %(here)s/database/beaker_sessions
|
||||
session_key = galaxysessions
|
||||
session_secret = changethisinproduction
|
||||
|
||||
# Configuration for debugging middleware
|
||||
#debug = False
|
||||
|
||||
@@ -88,4 +91,4 @@ paste.app_factory = galaxy.webapps.reports.buildapp:app_factory
|
||||
# static_style_dir = %(here)s/static/june_2007_style/blue
|
||||
|
||||
# Sentry (getsentry.com) DSN for catching bugs.
|
||||
#sentry_dsn = None
|
||||
#sentry_dsn = None
|
||||
+4
-1
@@ -119,6 +119,9 @@ paste.app_factory = galaxy.webapps.tool_shed.buildapp:app_factory
|
||||
# Check for WSGI compliance.
|
||||
#use_lint = False
|
||||
|
||||
# Intercept print statements and show them on the returned page.
|
||||
#use_printdebug = True
|
||||
|
||||
# NEVER enable this on a public site (even test or QA)
|
||||
#use_interactive = true
|
||||
|
||||
@@ -175,4 +178,4 @@ paste.app_factory = galaxy.webapps.tool_shed.buildapp:app_factory
|
||||
#static_style_dir = static/style/blue
|
||||
|
||||
# Sentry (getsentry.com) DSN for catching bugs.
|
||||
#sentry_dsn = None
|
||||
#sentry_dsn = None
|
||||
@@ -0,0 +1,130 @@
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
import yaml
|
||||
|
||||
from galaxy.webapps.config_manage import main
|
||||
|
||||
THIS_DIR = os.path.dirname(__file__)
|
||||
|
||||
|
||||
def test_reports_conversion_1607_sample():
|
||||
with _config_directory("1607_root_samples") as config_dir:
|
||||
config_dir.manage_cli(["convert", "reports"])
|
||||
config_dir.assert_not_exists("config/reports.ini")
|
||||
config_dir.assert_is_yaml("config/reports.yml")
|
||||
config_dir.assert_moved("config/reports.ini", "config/reports.ini.backup")
|
||||
with config_dir.open("config/reports.yml") as f:
|
||||
config = yaml.load(f)
|
||||
assert "reports" in config
|
||||
reports_config = config["reports"] or {}
|
||||
assert "use_beaker_session" not in reports_config
|
||||
|
||||
assert "uwsgi" in config
|
||||
uwsgi_config = config["uwsgi"]
|
||||
assert uwsgi_config["module"] == "galaxy.webapps.reports.buildapp:uwsgi_app()"
|
||||
|
||||
|
||||
def test_reports_build_sample():
|
||||
with _config_directory("1607_root_samples") as config_dir:
|
||||
config_dir.assert_not_exists("config/reports.yml.sample")
|
||||
config_dir.manage_cli(["build_sample_yaml", "reports", "--add-comments"])
|
||||
config_dir.assert_is_yaml("config/reports.yml.sample")
|
||||
|
||||
|
||||
def test_shed_conversion_1607_sample():
|
||||
with _config_directory("1607_root_samples") as config_dir:
|
||||
config_dir.manage_cli(["convert", "tool_shed"])
|
||||
config_dir.assert_not_exists("config/tool_shed.ini")
|
||||
config_dir.assert_is_yaml("config/tool_shed.yml")
|
||||
config_dir.assert_moved("config/tool_shed.ini", "config/tool_shed.ini.backup")
|
||||
|
||||
|
||||
def test_shed_build_sample():
|
||||
with _config_directory("1607_root_samples") as config_dir:
|
||||
config_dir.assert_not_exists("config/tool_shed.yml.sample")
|
||||
config_dir.manage_cli(["build_sample_yaml", "tool_shed", "--add-comments"])
|
||||
config_dir.assert_is_yaml("config/tool_shed.yml.sample")
|
||||
|
||||
|
||||
def test_shed_conversion_1607_prefix():
|
||||
with _config_directory("1607_root_filters") as config_dir:
|
||||
config_dir.manage_cli(["convert", "tool_shed"])
|
||||
config_dir.assert_not_exists("config/tool_shed.ini")
|
||||
config_dir.assert_is_yaml("config/tool_shed.yml")
|
||||
config_dir.assert_moved("config/tool_shed.ini", "config/tool_shed.ini.backup")
|
||||
with config_dir.open("config/tool_shed.yml") as f:
|
||||
config = yaml.load(f)
|
||||
assert "uwsgi" in config
|
||||
uwsgi_config = config["uwsgi"]
|
||||
assert "module" not in uwsgi_config
|
||||
assert uwsgi_config["mount"].startswith("/shed=galaxy.")
|
||||
|
||||
|
||||
def test_reports_conversion_1607_gzip():
|
||||
with _config_directory("1607_root_filters") as config_dir:
|
||||
config_dir.manage_cli(["convert", "reports"])
|
||||
config_dir.assert_not_exists("config/reports.ini")
|
||||
config_dir.assert_is_yaml("config/reports.yml")
|
||||
config_dir.assert_moved("config/reports.ini", "config/reports.ini.backup")
|
||||
with config_dir.open("config/reports.yml") as f:
|
||||
config = yaml.load(f)
|
||||
assert "uwsgi" in config
|
||||
uwsgi_config = config["uwsgi"]
|
||||
assert uwsgi_config["http-auto-gzip"] is True, uwsgi_config
|
||||
|
||||
|
||||
def test_build_uwsgi_yaml():
|
||||
with _config_directory("1607_root_samples") as config_dir:
|
||||
config_dir.manage_cli(["build_uwsgi_yaml"])
|
||||
|
||||
|
||||
class _TestConfigDirectory(object):
|
||||
|
||||
def __init__(self, base_name):
|
||||
temp_directory = tempfile.mkdtemp()
|
||||
os.removedirs(temp_directory)
|
||||
source_dir = os.path.join(THIS_DIR, base_name)
|
||||
shutil.copytree(source_dir, temp_directory)
|
||||
self.source_dir = source_dir
|
||||
self.temp_directory = temp_directory
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, type, value, tb):
|
||||
shutil.rmtree(self.temp_directory)
|
||||
|
||||
def assert_exists(self, path):
|
||||
assert os.path.exists(os.path.join(self.temp_directory, path))
|
||||
|
||||
def assert_not_exists(self, path):
|
||||
assert not os.path.exists(os.path.join(self.temp_directory, path))
|
||||
|
||||
def assert_is_yaml(self, path):
|
||||
self.assert_exists(path)
|
||||
with self.open(path, "r") as f:
|
||||
return yaml.load(f)
|
||||
|
||||
def open(self, path, *args):
|
||||
return open(os.path.join(self.temp_directory, path), *args)
|
||||
|
||||
def assert_moved(self, from_path, to_path):
|
||||
with open(os.path.join(self.source_dir, from_path), "r") as f:
|
||||
source_contents = f.read()
|
||||
|
||||
with self.open(to_path, "r") as f:
|
||||
dest_contents = f.read()
|
||||
|
||||
assert source_contents == dest_contents
|
||||
|
||||
def manage_cli(self, argv):
|
||||
real_argv = ["--galaxy_root", self.temp_directory]
|
||||
real_argv.extend(argv)
|
||||
main(real_argv)
|
||||
|
||||
|
||||
def _config_directory(base_name):
|
||||
return _TestConfigDirectory(base_name)
|
||||
@@ -81,6 +81,8 @@ def add_file( dataset, registry, json_file, output_path ):
|
||||
link_data_only = dataset.get( 'link_data_only', 'copy_files' )
|
||||
in_place = dataset.get( 'in_place', True )
|
||||
purge_source = dataset.get( 'purge_source', True )
|
||||
check_content = dataset.get( 'check_content' , True )
|
||||
auto_decompress = dataset.get( 'auto_decompress', True )
|
||||
try:
|
||||
ext = dataset.file_type
|
||||
except AttributeError:
|
||||
@@ -132,11 +134,11 @@ def add_file( dataset, registry, json_file, output_path ):
|
||||
ext = dataset.file_type
|
||||
else:
|
||||
# See if we have a gzipped file, which, if it passes our restrictions, we'll uncompress
|
||||
is_gzipped, is_valid = check_gzip( dataset.path )
|
||||
is_gzipped, is_valid = check_gzip( dataset.path, check_content=check_content )
|
||||
if is_gzipped and not is_valid:
|
||||
file_err( 'The gzipped uploaded file contains inappropriate content', dataset, json_file )
|
||||
return
|
||||
elif is_gzipped and is_valid:
|
||||
elif is_gzipped and is_valid and auto_decompress:
|
||||
if link_data_only == 'copy_files':
|
||||
# We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format
|
||||
CHUNK_SIZE = 2 ** 20 # 1Mb
|
||||
@@ -165,11 +167,11 @@ def add_file( dataset, registry, json_file, output_path ):
|
||||
data_type = 'gzip'
|
||||
if not data_type and bz2 is not None:
|
||||
# See if we have a bz2 file, much like gzip
|
||||
is_bzipped, is_valid = check_bz2( dataset.path )
|
||||
is_bzipped, is_valid = check_bz2( dataset.path, check_content )
|
||||
if is_bzipped and not is_valid:
|
||||
file_err( 'The gzipped uploaded file contains inappropriate content', dataset, json_file )
|
||||
return
|
||||
elif is_bzipped and is_valid:
|
||||
elif is_bzipped and is_valid and auto_decompress:
|
||||
if link_data_only == 'copy_files':
|
||||
# We need to uncompress the temp_name file
|
||||
CHUNK_SIZE = 2 ** 20 # 1Mb
|
||||
@@ -199,7 +201,7 @@ def add_file( dataset, registry, json_file, output_path ):
|
||||
if not data_type:
|
||||
# See if we have a zip archive
|
||||
is_zipped = check_zip( dataset.path )
|
||||
if is_zipped:
|
||||
if is_zipped and auto_decompress:
|
||||
if link_data_only == 'copy_files':
|
||||
CHUNK_SIZE = 2 ** 20 # 1Mb
|
||||
uncompressed = None
|
||||
@@ -265,7 +267,7 @@ def add_file( dataset, registry, json_file, output_path ):
|
||||
parts = dataset.name.split( "." )
|
||||
if len( parts ) > 1:
|
||||
ext = parts[-1].strip().lower()
|
||||
if not Binary.is_ext_unsniffable(ext):
|
||||
if check_content and not Binary.is_ext_unsniffable(ext):
|
||||
file_err( 'The uploaded binary file contains inappropriate content', dataset, json_file )
|
||||
return
|
||||
elif Binary.is_ext_unsniffable(ext) and dataset.file_type != ext:
|
||||
@@ -274,7 +276,7 @@ def add_file( dataset, registry, json_file, output_path ):
|
||||
return
|
||||
if not data_type:
|
||||
# We must have a text file
|
||||
if check_html( dataset.path ):
|
||||
if check_content and check_html( dataset.path ):
|
||||
file_err( 'The uploaded file contains inappropriate HTML content', dataset, json_file )
|
||||
return
|
||||
if data_type != 'binary':
|
||||
@@ -297,6 +299,8 @@ def add_file( dataset, registry, json_file, output_path ):
|
||||
ext = dataset.file_type
|
||||
data_type = ext
|
||||
# Save job info for the framework
|
||||
if ext == 'auto' and data_type == 'binary':
|
||||
ext = 'data'
|
||||
if ext == 'auto' and dataset.ext:
|
||||
ext = dataset.ext
|
||||
if ext == 'auto':
|
||||
@@ -336,8 +340,7 @@ def add_file( dataset, registry, json_file, output_path ):
|
||||
if dataset.get('uuid', None) is not None:
|
||||
info['uuid'] = dataset.get('uuid')
|
||||
json_file.write( dumps( info ) + "\n" )
|
||||
|
||||
if link_data_only == 'copy_files' and datatype.dataset_content_needs_grooming( output_path ):
|
||||
if link_data_only == 'copy_files' and datatype and datatype.dataset_content_needs_grooming( output_path ):
|
||||
# Groom the dataset content if necessary
|
||||
datatype.groom_dataset_content( output_path )
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@
|
||||
Pull Request 171 for more information. -->
|
||||
<param name="uuid" type="hidden" required="False" />
|
||||
<param name="to_posix_lines" type="hidden" value="Yes" />
|
||||
<param name="auto_decompress" type="hidden" value="Yes" />
|
||||
<!--
|
||||
<param name="to_posix_lines" type="select" display="checkboxes" multiple="True" label="Convert universal line endings to Posix line endings" help="Turn this option off if you upload a gzip, bz2 or zip archive which contains a binary file." value="Yes">
|
||||
<option value="Yes" selected="true">Yes</option>
|
||||
|
||||
Reference in New Issue
Block a user