From 470aa83d474d798e78b1361ccf48935b2696c6f5 Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Sat, 9 Mar 2019 02:27:23 +0100 Subject: [PATCH 001/324] Add support for Okta OIDC authentication. This is dependent on https://github.com/python-social-auth/social-core/pull/333. --- .../galaxy/scripts/components/login/Login.vue | 13 +++++---- config/oidc_backends_config.xml.sample | 11 ++++++++ lib/galaxy/authnz/managers.py | 22 +++++++++++++-- lib/galaxy/authnz/psa_authnz.py | 27 +++++++++++++++---- lib/galaxy/config.py | 1 + .../pipfiles/default/pinned-requirements.txt | 2 +- lib/galaxy/managers/configuration.py | 1 + 7 files changed, 64 insertions(+), 13 deletions(-) diff --git a/client/galaxy/scripts/components/login/Login.vue b/client/galaxy/scripts/components/login/Login.vue index fc7d84f9c2f..79d96c59f01 100644 --- a/client/galaxy/scripts/components/login/Login.vue +++ b/client/galaxy/scripts/components/login/Login.vue @@ -24,8 +24,8 @@ - - Sign in with Google + + Sign in with {{ idp.charAt(0).toUpperCase() + idp.slice(1) }}
@@ -56,6 +56,7 @@ export default { }, data() { let galaxy = getGalaxyInstance(); + let oidc_idps = Object.keys(galaxy.config.oidc).filter(function(key) { return galaxy.config.oidc[key]; }); return { login: null, password: null, @@ -65,7 +66,9 @@ export default { messageVariant: null, redirect: galaxy.params.redirect, session_csrf_token: galaxy.session_csrf_token, - enable_oidc: galaxy.config.enable_oidc + enable_oidc: galaxy.config.enable_oidc, + oidc_idps: oidc_idps, + oidc_idps_icons_class: {'google': 'fa fa-google', 'okta': 'fa fa-circle-o'} }; }, computed: { @@ -101,10 +104,10 @@ export default { this.messageText = message || "Login failed for an unknown reason."; }); }, - submitOIDCLogin: function(method) { + submitOIDCLogin: function(idp) { let rootUrl = getAppRoot(); axios - .post(`${rootUrl}authnz/google/login`) + .post(`${rootUrl}authnz/${idp}/login`) .then(response => { if (response.data.redirect_uri) { window.location = encodeURI(response.data.redirect_uri); diff --git a/config/oidc_backends_config.xml.sample b/config/oidc_backends_config.xml.sample index eee3c02daf2..4a99bbb7b16 100644 --- a/config/oidc_backends_config.xml.sample +++ b/config/oidc_backends_config.xml.sample @@ -22,4 +22,15 @@ login to Galaxy using their Google account. --> + + ... + ... + http://localhost:8080/authnz/okta/callback + + https://dev-000000.oktapreview.com/oauth2/default + + diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index d855387d074..076e3851f92 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -39,6 +39,7 @@ class AuthnzManager(object): :param config: sets the path for OIDC configuration file (e.g., oidc_backends_config.xml). """ + self.app = app self._parse_oidc_config(oidc_config_file) self._parse_oidc_backends_config(oidc_backends_config_file) @@ -88,8 +89,15 @@ class AuthnzManager(object): log.error("Could not find a node attribute 'name'; skipping the node '{}'.".format(child.tag)) continue idp = child.get('name').lower() - if idp == 'google': - self.oidc_backends_config[idp] = self._parse_google_config(child) + + idp_provider = { + "google": self._parse_google_config, + "okta": self._parse_okta_config + } + if idp in idp_provider: + self.oidc_backends_config[idp] = idp_provider[idp](child) + self.app.config.oidc[idp] = True + if len(self.oidc_backends_config) == 0: raise ParseError("No valid provider configuration parsed.") except ImportError: @@ -106,6 +114,16 @@ class AuthnzManager(object): rtv['prompt'] = config_xml.find('prompt').text return rtv + def _parse_okta_config(self, config_xml): + rtv = { + 'client_id': config_xml.find('client_id').text, + 'client_secret': config_xml.find('client_secret').text, + 'redirect_uri': config_xml.find('redirect_uri').text, + 'api_url': config_xml.find('api_url').text} + if config_xml.find('prompt') is not None: + rtv['prompt'] = config_xml.find('prompt').text + return rtv + def _unify_provider_name(self, provider): if provider.lower() in self.oidc_backends_config: return provider.lower() diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index c4448ec3b27..ea768983c24 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -18,11 +18,13 @@ DEFAULTS = { } BACKENDS = { - 'google': 'social_core.backends.google_openidconnect.GoogleOpenIdConnect' + 'google': 'social_core.backends.google_openidconnect.GoogleOpenIdConnect', + 'okta': 'social_core.backends.okta.OktaOpenIdConnect' } BACKENDS_NAME = { - 'google': 'google-openidconnect' + 'google': 'google-openidconnect', + 'okta': 'okta-openidconnect' } AUTH_PIPELINE = ( @@ -96,8 +98,12 @@ class PSAAuthnz(IdentityProvider): # the just logged-in user. self.config[setting_name('INACTIVE_USER_LOGIN')] = True - if provider == 'google': - self._setup_google_backend(oidc_backend_config) + idp_provider = { + "google": self._setup_google_backend, + "okta": self._setup_okta_backend + } + if provider in idp_provider: + idp_provider[provider](oidc_backend_config) def _setup_google_backend(self, oidc_backend_config): self.config[setting_name('AUTH_EXTRA_ARGUMENTS')] = {'access_type': 'offline'} @@ -107,6 +113,15 @@ class PSAAuthnz(IdentityProvider): if oidc_backend_config.get('prompt') is not None: self.config[setting_name('AUTH_EXTRA_ARGUMENTS')]['prompt'] = oidc_backend_config.get('prompt') + def _setup_okta_backend(self, oidc_backend_config): + self.config[setting_name('AUTH_EXTRA_ARGUMENTS')] = {'access_type': 'offline'} + self.config['SOCIAL_AUTH_OKTA_OPENIDCONNECT_KEY'] = oidc_backend_config.get('client_id') + self.config['SOCIAL_AUTH_OKTA_OPENIDCONNECT_SECRET'] = oidc_backend_config.get('client_secret') + self.config['SOCIAL_AUTH_OKTA_OPENIDCONNECT_API_URL'] = oidc_backend_config.get('api_url') + self.config['redirect_uri'] = oidc_backend_config.get('redirect_uri') + if oidc_backend_config.get('prompt') is not None: + self.config[setting_name('AUTH_EXTRA_ARGUMENTS')]['prompt'] = oidc_backend_config.get('prompt') + def _get_helper(self, name, do_import=False): this_config = self.config.get(setting_name(name), DEFAULTS.get(name, None)) return do_import and module_member(this_config) or this_config @@ -160,7 +175,9 @@ class Strategy(BaseStrategy): self.session = session if session else {} self.config = config self.config['SOCIAL_AUTH_REDIRECT_IS_HTTPS'] = True if self.request and self.request.host.startswith('https:') else False - self.config['SOCIAL_AUTH_GOOGLE_OPENIDCONNECT_EXTRA_DATA'] = ['id_token'] + if self.config['provider'] == "google": + self.config['SOCIAL_AUTH_GOOGLE_OPENIDCONNECT_EXTRA_DATA'] = ['id_token'] + super(Strategy, self).__init__(storage, tpl) def get_setting(self, name): diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index c270443c37d..4818bd17c31 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -212,6 +212,7 @@ class Configuration(object): self.enable_oidc = kwargs.get("enable_oidc", False) self.oidc_config = kwargs.get("oidc_config_file", self.oidc_config_file) self.oidc_backends_config = kwargs.get("oidc_backends_config_file", self.oidc_backends_config_file) + self.oidc = {} # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution # and moved to the tool shed. self.integrated_tool_panel_config = resolve_path(kwargs.get('integrated_tool_panel_config', 'integrated_tool_panel.xml'), self.root) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 59c0e44ab18..70204904082 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -139,7 +139,7 @@ routes==2.4.1 s3transfer==0.1.13 simplejson==3.16.0 six==1.11.0 -social-auth-core[openidconnect]==1.5.0 +social-auth-core[openidconnect]==3.1.0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.33.11 sqlalchemy==1.2.18 diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index bd3d7466183..7a5e70e9d69 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -63,6 +63,7 @@ class ConfigSerializer(base.ModelSerializer): 'allow_user_creation' : _defaults_to(False), 'use_remote_user' : _defaults_to(None), 'enable_oidc' : _defaults_to(False), + 'oidc' : _defaults_to(self.app.config.oidc), 'enable_quotas' : _defaults_to(False), 'remote_user_logout_href' : _defaults_to(''), 'datatypes_disable_auto' : _defaults_to(False), From 5f82d224174069b47aa951dc69cf9c223ca4fd48 Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Sat, 9 Mar 2019 02:31:22 +0100 Subject: [PATCH 002/324] Update documentation for Okta --- doc/source/admin/authentication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/admin/authentication.md b/doc/source/admin/authentication.md index bae80da8d92..bf43f79b982 100644 --- a/doc/source/admin/authentication.md +++ b/doc/source/admin/authentication.md @@ -18,7 +18,7 @@ If deploying Galaxy using the default authentication option, user activation can ## OIDC and OAuth2.0 Leveraging OpenID Connect (OIDC) protocol, we enable login to Galaxy without explicitly creating a Galaxy user. This feature is disabled by default. In short, to enable this feature, a Galaxy server admin has to take the following two steps: -1. Define the Galaxy instance on an OIDC identity provider. At the moment, we support Google. To set a Galaxy instance on Google, go to _credentials_ section at [developers console](https://console.developers.google.com/), and configure the instance. At the end, you'll receive _client ID_ and _client secret_ take a note of these two tokens. +1. Define the Galaxy instance on an OIDC identity provider. At the moment, we support Google and Okta. To set a Galaxy instance on Google, go to _credentials_ section at [developers console](https://console.developers.google.com/), and configure the instance. At the end, you'll receive _client ID_ and _client secret_ take a note of these two tokens. For Okta, create a new application in Okta, type _web_. At the end you should take note of the _client ID_ and _client secret_ tokens. 2. Configure Galaxy. In the `galaxy.yml` file enable the OIDC service using the `enable_oidc` key and set the two configuration files (i.e., `oidc_config_file` and `oidc_backends_config_file`), based on the IdP information. From aa732e581085874f17878c5a02136a47054333cc Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Sat, 9 Mar 2019 18:38:00 +0100 Subject: [PATCH 003/324] OIDC configuration, change dependencies --- lib/galaxy/dependencies/__init__.py | 3 +++ lib/galaxy/dependencies/conditional-requirements.txt | 3 +++ .../dependencies/pipfiles/default/pinned-requirements.txt | 1 - 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 073cf67301e..5249bf0392d 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -146,6 +146,9 @@ class ConditionalDependencies(object): ('docker' in self.container_interface_types or 'docker_swarm' in self.container_interface_types)) + def check_social_auth_core(self): + return self.config.get("enable_oidc", False) + def optional(config_file=None): if not config_file: diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index b406a2e03e4..c99684eef9b 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -24,3 +24,6 @@ pykube==0.15.0 kamaki watchdog + +# OIDC dependencies +social-auth-core[openidconnect]==3.1.0 \ No newline at end of file diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 70204904082..c72da5b4cdc 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -139,7 +139,6 @@ routes==2.4.1 s3transfer==0.1.13 simplejson==3.16.0 six==1.11.0 -social-auth-core[openidconnect]==3.1.0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.33.11 sqlalchemy==1.2.18 From 4a5922139a129cb4f5f2ca0a01c739cdb3f2bd53 Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Tue, 12 Mar 2019 20:46:38 +0100 Subject: [PATCH 004/324] Updates for Okta IdP and general IdPs --- client/galaxy/scripts/components/login/Login.vue | 7 +++++++ lib/galaxy/authnz/managers.py | 9 +++++++++ lib/galaxy/authnz/psa_authnz.py | 6 +++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/client/galaxy/scripts/components/login/Login.vue b/client/galaxy/scripts/components/login/Login.vue index 79d96c59f01..3363b9e1538 100644 --- a/client/galaxy/scripts/components/login/Login.vue +++ b/client/galaxy/scripts/components/login/Login.vue @@ -57,6 +57,13 @@ export default { data() { let galaxy = getGalaxyInstance(); let oidc_idps = Object.keys(galaxy.config.oidc).filter(function(key) { return galaxy.config.oidc[key]; }); + // Icons to use for each IdP + let oidc_idps_icons = {'google': 'fa fa-google', 'okta': 'fa fa-circle-o'}; + // Add default icons to IdPs without icons + oidc_idps.filter(function(key) { return oidc_idps_icons[key] === undefined; }).forEach(function(idp) { + oidc_idps_icons[idp] = 'fa fa-id-card' + }); + return { login: null, password: null, diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 08a8b09634e..f6b657e3c60 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -39,6 +39,7 @@ class AuthnzManager(object): :param config: sets the path for OIDC configuration file (e.g., oidc_backends_config.xml). """ + self.app = app self._parse_oidc_config(oidc_config_file) self._parse_oidc_backends_config(oidc_backends_config_file) @@ -90,6 +91,8 @@ class AuthnzManager(object): idp = child.get('name').lower() if idp in BACKENDS_NAME: self.oidc_backends_config[idp] = self._parse_idp_config(child) + # Add this variable so we can dynamically show OIDC IdP in Vue template + self.app.config.oidc[idp] = True if len(self.oidc_backends_config) == 0: raise ParseError("No valid provider configuration parsed.") except ImportError: @@ -102,8 +105,14 @@ class AuthnzManager(object): 'client_id': config_xml.find('client_id').text, 'client_secret': config_xml.find('client_secret').text, 'redirect_uri': config_xml.find('redirect_uri').text} + if config_xml.find('prompt') is not None: rtv['prompt'] = config_xml.find('prompt').text + if config_xml.find('api_url') is not None: + rtv['api_url'] = config_xml.find('api_url').text + if config_xml.find('url') is not None: + rtv['url'] = config_xml.find('url').text + return rtv def _unify_provider_name(self, provider): diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index c5170a0b61b..106672426e4 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -86,7 +86,7 @@ DISCONNECT_PIPELINE = ( class PSAAuthnz(IdentityProvider): def __init__(self, provider, oidc_config, oidc_backend_config): self.config = {'provider': provider.lower()} - for key, value in oidc_config.iteritems(): + for key, value in oidc_config.items(): self.config[setting_name(key)] = value self.config[setting_name('USER_MODEL')] = 'models.User' @@ -110,6 +110,10 @@ class PSAAuthnz(IdentityProvider): self.config['redirect_uri'] = oidc_backend_config.get('redirect_uri') if oidc_backend_config.get('prompt') is not None: self.config[setting_name('AUTH_EXTRA_ARGUMENTS')]['prompt'] = oidc_backend_config.get('prompt') + if oidc_backend_config.get('api_url') is not None: + self.config[setting_name('API_URL')] = oidc_backend_config.get('api_url') + if oidc_backend_config.get('url') is not None: + self.config[setting_name('URL')] = oidc_backend_config.get('url') def _get_helper(self, name, do_import=False): this_config = self.config.get(setting_name(name), DEFAULTS.get(name, None)) From d4c33f7386146873b111960f9d49a34540f727a3 Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Tue, 12 Mar 2019 20:56:37 +0100 Subject: [PATCH 005/324] social-auth-core should be pinned... can't be optional right now --- lib/galaxy/dependencies/conditional-requirements.txt | 3 --- .../dependencies/pipfiles/default/pinned-requirements.txt | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 2f6653357e9..b406a2e03e4 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -24,6 +24,3 @@ pykube==0.15.0 kamaki watchdog - -# OIDC dependencies -social-auth-core[openidconnect]==3.1.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index a6792e1c7dd..96c1eae6166 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -140,6 +140,7 @@ rsa==4.0 s3transfer==0.1.13 simplejson==3.16.0 six==1.11.0 +social-auth-core[openidconnect]==3.1.0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.33.11 sqlalchemy==1.2.18 From 993d899e9c5c4f73ab5fe43e755347af9c6bf5d6 Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Wed, 17 Jul 2019 22:47:40 +0200 Subject: [PATCH 006/324] Add Okta to oidc_backends_config.xml.sample --- .../config/sample/oidc_backends_config.xml.sample | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/galaxy/config/sample/oidc_backends_config.xml.sample b/lib/galaxy/config/sample/oidc_backends_config.xml.sample index 90b157b1632..c4b832348f8 100644 --- a/lib/galaxy/config/sample/oidc_backends_config.xml.sample +++ b/lib/galaxy/config/sample/oidc_backends_config.xml.sample @@ -108,4 +108,14 @@ Please mind `http` and `https`. consent + + ... + ... + http://localhost:8080/authnz/okta/callback + + https://${company}.okta.com/oauth2 + From 341d555ad68ddf55a4b0b894ee3bdc7b16d0f397 Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Wed, 17 Jul 2019 22:53:12 +0200 Subject: [PATCH 007/324] Add documentation on how to get the application set up in Okta. --- .../config/sample/oidc_backends_config.xml.sample | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/config/sample/oidc_backends_config.xml.sample b/lib/galaxy/config/sample/oidc_backends_config.xml.sample index c4b832348f8..18ae8b18170 100644 --- a/lib/galaxy/config/sample/oidc_backends_config.xml.sample +++ b/lib/galaxy/config/sample/oidc_backends_config.xml.sample @@ -115,7 +115,18 @@ Please mind `http` and `https`. - https://${company}.okta.com/oauth2 + ... From 9400f26eeffb54e25456966fb4d9df5468d89712 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 6 Sep 2019 22:55:03 +0200 Subject: [PATCH 008/324] move ITs to tools/interactive (cherry picked from commit 1c5bc4f55a781e48e4fd6017e4e49a52db8c4657) --- interactivetool_askomics.xml | 59 ++++++++++++++++ interactivetool_bam_iobio.xml | 44 ++++++++++++ interactivetool_cellxgene.xml | 29 ++++++++ interactivetool_ethercalc.xml | 63 +++++++++++++++++ interactivetool_hicbrowser.xml | 33 +++++++++ interactivetool_jupyter_notebook.xml | 100 +++++++++++++++++++++++++++ interactivetool_neo4j.xml | 41 +++++++++++ interactivetool_phinch.xml | 37 ++++++++++ 8 files changed, 406 insertions(+) create mode 100644 interactivetool_askomics.xml create mode 100644 interactivetool_bam_iobio.xml create mode 100644 interactivetool_cellxgene.xml create mode 100644 interactivetool_ethercalc.xml create mode 100644 interactivetool_hicbrowser.xml create mode 100644 interactivetool_jupyter_notebook.xml create mode 100644 interactivetool_neo4j.xml create mode 100644 interactivetool_phinch.xml diff --git a/interactivetool_askomics.xml b/interactivetool_askomics.xml new file mode 100644 index 00000000000..71c3fe9a437 --- /dev/null +++ b/interactivetool_askomics.xml @@ -0,0 +1,59 @@ + + AskOmics, a visual SPARQL query builder + + quay.io/askomics/askomics-ie:17.12_g19.09 + + + + 6543 + /login_api_gie?key=abcd + + + + ${__app__.config.galaxy_infrastructure_url} + + #if $__user__: + #for $api_key in $__user__.api_keys: + ${api_key.key} + #break + #end for + #end if + + + + + + + + + + + + + + AskOmics is a visual SPARQL query interface supporting both intuitive data integration and + querying while shielding the user from most of the technical difficulties underlying RDF and SPARQL. + + diff --git a/interactivetool_bam_iobio.xml b/interactivetool_bam_iobio.xml new file mode 100644 index 00000000000..2f1c531861c --- /dev/null +++ b/interactivetool_bam_iobio.xml @@ -0,0 +1,44 @@ + + + qiaoy/iobio-bundle.bam-iobio:1.0-ondemand + + + + 80 + + + + /tmp/app.conf && + mv /tmp/app.conf /etc/supervisor.d/app.conf && + + /usr/bin/supervisord -c /etc/supervisord.conf + ]]> + + + + + + + + + + + BAM iobio visualisation. + + diff --git a/interactivetool_cellxgene.xml b/interactivetool_cellxgene.xml new file mode 100644 index 00000000000..546bd7ab58f --- /dev/null +++ b/interactivetool_cellxgene.xml @@ -0,0 +1,29 @@ + + + quay.io/galaxy/cellxgene-galaxy-ie:ie2 + + + + 80 + + + + + + + + + + + + + + Interactive tool for visualising AnnData. + + diff --git a/interactivetool_ethercalc.xml b/interactivetool_ethercalc.xml new file mode 100644 index 00000000000..88676c1e997 --- /dev/null +++ b/interactivetool_ethercalc.xml @@ -0,0 +1,63 @@ + + + shiltemann/ethercalc-galaxy-ie:17.05 + + + + 8000 + + + loading.txt + && + curl --include --request PUT --header "Content-Type: text/csv" --data-binary @loading.txt http://localhost:8000/_/galaxy + && + + ## remove dump file so this doesnt appear in audit trail + rm /dump.json + && + + ## load dataset into worksheet + curl --include --request PUT --header "Content-Type: text/csv" --data-binary @$infile http://localhost:8000/_/galaxy + && + + tail -f /etc/hosts + + ]]> + + + +&1) +while [[ \${STATUS} =~ "refused" ]] +do + echo "waiting for ethercalc: \$STATUS \n" + STATUS=\$(curl --include 'http://localhost:8000/_/galaxy' 2>&1) + sleep 2 +done + ]]> + + + + + + + + + + + + EtherCalc is a web spreadsheet. + https://ethercalc.net + + diff --git a/interactivetool_hicbrowser.xml b/interactivetool_hicbrowser.xml new file mode 100644 index 00000000000..7aaa8a3e8e7 --- /dev/null +++ b/interactivetool_hicbrowser.xml @@ -0,0 +1,33 @@ + + + bgruening/hicbrowser + + + + 80 + + + + + + + + + + + + + + Visualising HiC data with HiCBrowser. + + diff --git a/interactivetool_jupyter_notebook.xml b/interactivetool_jupyter_notebook.xml new file mode 100644 index 00000000000..9dd3e36d3ee --- /dev/null +++ b/interactivetool_jupyter_notebook.xml @@ -0,0 +1,100 @@ + + + quay.io/bgruening/docker-jupyter-notebook:ie2 + + + + 8888 + ipython/tree + + + + ${__app__.security.encode_id($jupyter_notebook.history_id)} + ${__app__.config.galaxy_infrastructure_url} + 8080 + ${__app__.config.galaxy_infrastructure_url} + + #if $__user__: + #for $api_key in $__user__.api_keys: + ${api_key.key} + #break + #end for + #end if + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations, + visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, + machine learning, and much more. + + Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to + do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. + + You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. + If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. + + You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. + + diff --git a/interactivetool_neo4j.xml b/interactivetool_neo4j.xml new file mode 100644 index 00000000000..0b31a8b303e --- /dev/null +++ b/interactivetool_neo4j.xml @@ -0,0 +1,41 @@ + + + quay.io/sanbi-sa/neo_ie:3.1.9 + + + + 80 + + + + + 2345 + 2345 + false + + + + + + + + + + + + + Neo4j is a highly scalable, robust native graph database. + + diff --git a/interactivetool_phinch.xml b/interactivetool_phinch.xml new file mode 100644 index 00000000000..d77c9274ad3 --- /dev/null +++ b/interactivetool_phinch.xml @@ -0,0 +1,37 @@ + + + shiltemann/docker-phinch-galaxy:16.04 + + + + 80 + + + &1 > /var/log/phinch.log + + ]]> + + + + + + + + + + + Interactive tool for visualising Biom data. + + From 2f67bb0f4d539b6f768edc635f01c207a5046947 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 6 Sep 2019 23:05:01 +0200 Subject: [PATCH 009/324] move default notebook as well (cherry picked from commit 6d8848e399446fdc3cb1e0c81b1e88c80b03c515) --- default_notebook.ipynb | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 default_notebook.ipynb diff --git a/default_notebook.ipynb b/default_notebook.ipynb new file mode 100644 index 00000000000..e9573752478 --- /dev/null +++ b/default_notebook.ipynb @@ -0,0 +1,53 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Welcome to the interactive Galaxy IPython Notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can access your data via the dataset number. Using a Python kernel, you can access dataset number 42 with ``handle = open(get(42), 'r')``.\n", + "To save data, write your data to a file, and then call ``put('filename.txt')``. The dataset will then be available in your galaxy history.\n
", + "When using a non-Python kernel, ``get`` and ``put`` are available as command-line tools, which can be accessed using system calls in R, Julia, and Ruby. For example, to read dataset number 42 into R, you can write ```handle <- file(system('get -i 42', intern = TRUE))```.\n", + "To save data in R, write the data to a file and then call ``system('put -p filename.txt')``.\n", + "Notebooks can be saved to Galaxy by clicking the large green button at the top right of the IPython interface.
\n", + "More help and informations can be found on the project [website](https://github.com/bgruening/docker-jupyter-notebook)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 58a00bef9c035495f46a1a92ac73dbe7fe66751e Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 6 Sep 2019 23:07:39 +0200 Subject: [PATCH 010/324] improve description of Askomics (cherry picked from commit 1d4dc27e509a4084857fec63c20698d69979c8eb) --- interactivetool_askomics.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_askomics.xml b/interactivetool_askomics.xml index 71c3fe9a437..c1ce9d3bd11 100644 --- a/interactivetool_askomics.xml +++ b/interactivetool_askomics.xml @@ -1,5 +1,5 @@ - AskOmics, a visual SPARQL query builder + a visual SPARQL query builder quay.io/askomics/askomics-ie:17.12_g19.09 From dbe1fac5e87a0105ae437e86491e8beeb5eed609 Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Tue, 24 Sep 2019 08:36:46 +0200 Subject: [PATCH 011/324] Update okta backend name --- lib/galaxy/authnz/psa_authnz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index 8dc0e04f53d..4502c1447d0 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -21,7 +21,7 @@ BACKENDS = { 'google': 'social_core.backends.google_openidconnect.GoogleOpenIdConnect', 'globus': 'social_core.backends.globus.GlobusOpenIdConnect', 'elixir': 'social_core.backends.elixir.ElixirOpenIdConnect', - 'okta': 'social_core.backends.okta.OktaOpenIdConnect' + 'okta': 'social_core.backends.okta_openidconnect.OktaOpenIdConnect' } BACKENDS_NAME = { From 06bbc492e2faf0d7c7d53d0ac207f34ca3224e21 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Thu, 3 Oct 2019 13:40:33 +0200 Subject: [PATCH 012/324] move to right dir --- interactivetool_paraview.xml | 50 ++++++++++++++++++++++++++ interactivetool_rstudio.xml | 69 ++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 interactivetool_paraview.xml create mode 100644 interactivetool_rstudio.xml diff --git a/interactivetool_paraview.xml b/interactivetool_paraview.xml new file mode 100644 index 00000000000..c801600ef0e --- /dev/null +++ b/interactivetool_paraview.xml @@ -0,0 +1,50 @@ + + + + bmcv/galaxy-paraviewweb:latest + + + + 8777 + + + + + localhost:8080 + wss + -dr,--mesa-swr + + + + + + + + + + + + + ParaView is an open-source, multi-platform application designed to visualize data sets of varying sizes from small to very large. + + diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml new file mode 100644 index 00000000000..30b55db7f11 --- /dev/null +++ b/interactivetool_rstudio.xml @@ -0,0 +1,69 @@ + + + quay.io/erasche/docker-rstudio-notebook:19.05 + + + + 80 + rstudio/ + + + + ${__app__.security.encode_id($jupyter_notebook.history_id)} + ${__app__.config.galaxy_infrastructure_url} + 8080 + ${__app__.config.galaxy_infrastructure_url} + + #if $__user__: + #for $api_key in $__user__.api_keys: + ${api_key.key} + #break + #end for + #end if + + + + + + + + + + + + + + + + + + + + This familiar R analysis software suite will let you explore your + datasets in depth. Comes with ggplot2, RODBC, maps, shinyapps, knitr, + LaTeX, bioconductor, cummeRbund, and many more pre-installed packages. + + Galaxy offers you to use RStudio directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to + do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in RStudio. + + The convenience functions gx_put() and gx_get() are available to you to interact with your current Galaxy history. You can save your workspace with gx_save(). + + For example, gx_get(42) will fetch dataset 42 from your history and return the file location + + From 6b0b5edbbfa092e9e51732d8e9ae931b22e197e2 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 5 Oct 2019 23:58:46 +0200 Subject: [PATCH 013/324] add wilson --- interactivetool_wilson.xml | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 interactivetool_wilson.xml diff --git a/interactivetool_wilson.xml b/interactivetool_wilson.xml new file mode 100644 index 00000000000..5ce1b73272f --- /dev/null +++ b/interactivetool_wilson.xml @@ -0,0 +1,51 @@ + + Webbased Interactive Omics visualization + + loosolab/wilson:2.1.0 + + + + 3838 + + + &1 + ]]> + + + + + + + + + + +`_ + +.. class:: infomark + +Wilson uses the CLARION file format, which is a generic file format for quantitative comparisons of high throughput screens. + +CLARION is a data format specially developed to be used with Wilson, which relies on a tab-delimited table with +a metadata header to describe the following columns. It is based on the Summarized Experiment format and supports +all types of data which can be reduced to features and their annotation (e.g. genes, transcripts, proteins, probes) +with assigned numerical values (e.g. count, score, log2foldchange, z-score, p-value). Most result tables derived from RNA-Seq, +ChIP/ATAC-Seq, Proteomics, Microarrays, and many other analyses can thus be easily reformatted to become compatible +without having to modify the code of Wilson for each specific experiment. + +Please check the following link for details considering the `CLARION format `_. + + + +]]> + + + 10.1093/bioinformatics/bty711 + + From 5df22293ef936aabe9d4b4c14e1afb88e2f77662 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 6 Oct 2019 00:01:57 +0200 Subject: [PATCH 014/324] deactivate monitor script --- interactivetool_rstudio.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml index 30b55db7f11..7454c973b85 100644 --- a/interactivetool_rstudio.xml +++ b/interactivetool_rstudio.xml @@ -36,6 +36,8 @@ ## change into the directory where the notebooks are located cd ./rstudio/ && + sed -i 's|/monitor.*||g' /etc/services.d/nginx/run && + /init ]]> From 73dd265703a1fa7b7c65be73aae22138f9fd2512 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 6 Oct 2019 00:48:50 +0200 Subject: [PATCH 015/324] fix tool id --- interactivetool_wilson.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_wilson.xml b/interactivetool_wilson.xml index 5ce1b73272f..9d724a0fbe5 100644 --- a/interactivetool_wilson.xml +++ b/interactivetool_wilson.xml @@ -1,4 +1,4 @@ - + Webbased Interactive Omics visualization loosolab/wilson:2.1.0 From c7f5136c8f7942312b2f83075b25e5ff847f9b82 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 8 Oct 2019 14:51:38 +0200 Subject: [PATCH 016/324] temporarily disable all input data --- interactivetool_rstudio.xml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml index 7454c973b85..9f39f227348 100644 --- a/interactivetool_rstudio.xml +++ b/interactivetool_rstudio.xml @@ -28,11 +28,6 @@ mkdir -p ./rstudio/outputs/ && mkdir -p ./rstudio/data && - #if $input: - #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) - ln -sf '$input' './rstudio/data/${cleaned_name}' && - #end if - ## change into the directory where the notebooks are located cd ./rstudio/ && @@ -43,7 +38,7 @@ ]]> - + @@ -60,12 +55,12 @@ This familiar R analysis software suite will let you explore your datasets in depth. Comes with ggplot2, RODBC, maps, shinyapps, knitr, LaTeX, bioconductor, cummeRbund, and many more pre-installed packages. - + Galaxy offers you to use RStudio directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in RStudio. - The convenience functions gx_put() and gx_get() are available to you to interact with your current Galaxy history. You can save your workspace with gx_save(). + The convenience functions gx_put() and gx_get() are available to you to interact with your current Galaxy history. You can save your workspace with gx_save(). - For example, gx_get(42) will fetch dataset 42 from your history and return the file location + For example, gx_get(42) will fetch dataset 42 from your history and return the file location From ae38c98f5adf246d4516ffe9248413099ef893aa Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 8 Oct 2019 14:52:54 +0200 Subject: [PATCH 017/324] add two more --- interactivetool_rstudio.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml index 9f39f227348..8338497f70c 100644 --- a/interactivetool_rstudio.xml +++ b/interactivetool_rstudio.xml @@ -13,6 +13,8 @@ ${__app__.config.galaxy_infrastructure_url} 8080 ${__app__.config.galaxy_infrastructure_url} + true + true #if $__user__: #for $api_key in $__user__.api_keys: From b92f4720273172cda4b64f96e0ce7e36e797893d Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 8 Oct 2019 14:53:31 +0200 Subject: [PATCH 018/324] also from prod --- interactivetool_jupyter_notebook.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/interactivetool_jupyter_notebook.xml b/interactivetool_jupyter_notebook.xml index 9dd3e36d3ee..60cdc00253f 100644 --- a/interactivetool_jupyter_notebook.xml +++ b/interactivetool_jupyter_notebook.xml @@ -39,7 +39,7 @@ ## copy default notebook cp '$__tool_directory__/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb && jupyter trust ./ipython_galaxy_notebook.ipynb && - jupyter lab --no-browser --NotebookApp.shutdown_button=True && + jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' #else: @@ -50,7 +50,7 @@ #if $mode.run_it: jupyter nbconvert --to notebook --execute --output ./ipython_galaxy_notebook.ipynb --allow-errors ./*.ipynb && #else: - jupyter lab --no-browser --NotebookApp.shutdown_button=True && + jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && #end if cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' #end if @@ -88,13 +88,13 @@ The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations, visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, machine learning, and much more. - + Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. - + You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. - + You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. From 0bd3376068fe8d9059628b8ac127f4473d16eaf0 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 8 Oct 2019 14:59:38 +0200 Subject: [PATCH 019/324] switch version --- interactivetool_rstudio.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml index 8338497f70c..d0b11257a3f 100644 --- a/interactivetool_rstudio.xml +++ b/interactivetool_rstudio.xml @@ -1,6 +1,6 @@ - quay.io/erasche/docker-rstudio-notebook:19.05 + quay.io/erasche/docker-rstudio-notebook:19.09 From 8b5d5bf5c5812f101ae00a7e82b9f3c537ae32bf Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 8 Oct 2019 16:38:15 +0200 Subject: [PATCH 020/324] chown, hacks --- interactivetool_rstudio.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml index d0b11257a3f..8f5d8792bf4 100644 --- a/interactivetool_rstudio.xml +++ b/interactivetool_rstudio.xml @@ -35,7 +35,11 @@ sed -i 's|/monitor.*||g' /etc/services.d/nginx/run && - /init + ##/init + rstudio-server start && + /etc/init.d/syslog-ng start && + chmod 777 /tmp -R && + tail -f /var/log/rstudio-server/rserver.log ]]> From ca0624a341464c4c9b4d9409d509ce3b8cf64ea3 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 8 Oct 2019 16:48:42 +0200 Subject: [PATCH 021/324] fix pot --- interactivetool_rstudio.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml index 8f5d8792bf4..f15e9047cf4 100644 --- a/interactivetool_rstudio.xml +++ b/interactivetool_rstudio.xml @@ -4,7 +4,7 @@ - 80 + 8787 rstudio/ From 48b5cb7bc803031832114c49ab4f2c899fb11c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 8 Oct 2019 20:09:27 +0200 Subject: [PATCH 022/324] latest RStudio changes --- interactivetool_rstudio.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml index f15e9047cf4..e6ee98af154 100644 --- a/interactivetool_rstudio.xml +++ b/interactivetool_rstudio.xml @@ -4,7 +4,7 @@ - 8787 + 80 rstudio/ @@ -35,9 +35,10 @@ sed -i 's|/monitor.*||g' /etc/services.d/nginx/run && - ##/init - rstudio-server start && /etc/init.d/syslog-ng start && + /init & + ##rstudio-server start && + sleep 5 && chmod 777 /tmp -R && tail -f /var/log/rstudio-server/rserver.log From d4403bc344cf2d4723bbdf9eefdef115141e56f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 8 Oct 2019 20:09:53 +0200 Subject: [PATCH 023/324] use lab entrypoint --- interactivetool_jupyter_notebook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_jupyter_notebook.xml b/interactivetool_jupyter_notebook.xml index 60cdc00253f..787d7fa64fa 100644 --- a/interactivetool_jupyter_notebook.xml +++ b/interactivetool_jupyter_notebook.xml @@ -5,7 +5,7 @@ 8888 - ipython/tree + ipython/lab From 4a558dba1ac0432b57b4bbd62b85adf8c5576a7d Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 8 Oct 2019 20:32:58 +0200 Subject: [PATCH 024/324] more reverts --- interactivetool_rstudio.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml index e6ee98af154..63e0eb3adb0 100644 --- a/interactivetool_rstudio.xml +++ b/interactivetool_rstudio.xml @@ -39,6 +39,7 @@ /init & ##rstudio-server start && sleep 5 && + chmod 777 /tmp -R && tail -f /var/log/rstudio-server/rserver.log From b050d84dbdeb31a95a35f0ab1ac84885daeb2718 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 9 Oct 2019 16:02:24 +0200 Subject: [PATCH 025/324] include env --- interactivetool_wilson.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/interactivetool_wilson.xml b/interactivetool_wilson.xml index 9d724a0fbe5..94abe38311d 100644 --- a/interactivetool_wilson.xml +++ b/interactivetool_wilson.xml @@ -8,10 +8,12 @@ 3838 + + feature_selection + &1 ]]> From 0dbfa7ce3282e003bc3158e622ee8e22815febe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 9 Oct 2019 23:35:41 +0200 Subject: [PATCH 026/324] use own container --- interactivetool_wilson.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_wilson.xml b/interactivetool_wilson.xml index 94abe38311d..302ff3b0e4a 100644 --- a/interactivetool_wilson.xml +++ b/interactivetool_wilson.xml @@ -1,7 +1,7 @@ Webbased Interactive Omics visualization - loosolab/wilson:2.1.0 + quay.io/bgruening/wilson-app From 88e3f31dea523ed69ebfc4fe64a5075dc7096f48 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Oct 2019 00:37:59 +0200 Subject: [PATCH 027/324] fix phinch IE --- interactivetool_phinch.xml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/interactivetool_phinch.xml b/interactivetool_phinch.xml index d77c9274ad3..382cf32f0a9 100644 --- a/interactivetool_phinch.xml +++ b/interactivetool_phinch.xml @@ -1,4 +1,4 @@ - + shiltemann/docker-phinch-galaxy:16.04 @@ -11,12 +11,14 @@ ## ToDo nginx, proxy.conf etc can be removed from the container #import os - #set $name = os.path.splitext(str($infile.display_name))[0] + #set $name = os.path.splitext(str($infile.display_name).replace(' ', '_'))[0] ## in case someone names the data testdata. rm /home/Phinch/data/testdata.biom | true && - ln -s '$infile' /home/Phinch/data/${name}.biom && + ln -s '$infile' '/home/Phinch/data/${name}.biom' && cd /home/Phinch/data && sed -i "s/'REPLACE_ME'/'${name}.biom'/g" /home/Phinch/scripts/readFile.js && + sed -i "s/http/https/g" /home/Phinch/scripts/readFile.js && + ## keep it running cd /home/Phinch && php -S 0.0.0.0:80 2>&1 > /var/log/phinch.log From edd4da31e3a8077424cc3524bc4e988c74a17ddc Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Oct 2019 12:16:45 +0200 Subject: [PATCH 028/324] Update Askomics. --- interactivetool_askomics.xml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/interactivetool_askomics.xml b/interactivetool_askomics.xml index c1ce9d3bd11..b4cb6bff0d1 100644 --- a/interactivetool_askomics.xml +++ b/interactivetool_askomics.xml @@ -1,17 +1,17 @@ - a visual SPARQL query builder + AskOmics, a visual SPARQL query builder - quay.io/askomics/askomics-ie:17.12_g19.09 + askomics/askomics-ie:19.01.3 6543 - /login_api_gie?key=abcd + /login_api_gie?key=${__user_name__} ${__app__.config.galaxy_infrastructure_url} - + #if $__user__: #for $api_key in $__user__.api_keys: ${api_key.key} @@ -19,15 +19,26 @@ #end for #end if + ${__user_name__} + ${__user_email__} + ${__user_name__} + From 30424102deb606e69b8e29505d722bdee298d821 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Oct 2019 16:35:42 +0200 Subject: [PATCH 029/324] make bam.io.bio work --- interactivetool_bam_iobio.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/interactivetool_bam_iobio.xml b/interactivetool_bam_iobio.xml index 2f1c531861c..6c78eaa335b 100644 --- a/interactivetool_bam_iobio.xml +++ b/interactivetool_bam_iobio.xml @@ -21,6 +21,9 @@ sed -i "s@\"wss://services.iobio.io/bamstatsalive/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/bamstatsalive/\"@" js/bam.iobio.js/bam.iobio.js && sed -i "s@\"wss://services.iobio.io/samheader/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/samheader/\"@" js/bam.iobio.js/bam.iobio.js && + + sed -i 's/deny all;//g' /etc/nginx/nginx.conf && + cp '${infile}' /input/bamfile.bam && cp '${infile.metadata.bam_index}' /input/bamfile.bam.bai && mkdir /var/log/supervisor/ && From fec1801d07540dccb0af8d20c71833f659b97961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20Preu=C3=9Fner?= Date: Mon, 14 Oct 2019 14:43:40 +0200 Subject: [PATCH 030/324] Handle env vars; updated container to latest tag --- interactivetool_wilson.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/interactivetool_wilson.xml b/interactivetool_wilson.xml index 302ff3b0e4a..c362a758c4e 100644 --- a/interactivetool_wilson.xml +++ b/interactivetool_wilson.xml @@ -1,7 +1,7 @@ Webbased Interactive Omics visualization - quay.io/bgruening/wilson-app + loosolab/wilson:2.1.1 @@ -10,9 +10,12 @@ feature_selection + true /home/shiny/.Renviron && + echo $WILSON_BLACKLIST_EXAMPLES >> /home/shiny/.Renviron && ln -s ${infile} /srv/shiny-server/external_data/input.clarion && exec shiny-server 2>&1 ]]> From a4f9346b67fbf35fef2c8211d323a5bfb5ba3d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20Preu=C3=9Fner?= Date: Wed, 16 Oct 2019 09:39:31 +0200 Subject: [PATCH 031/324] Fixed bug in command for .Renviron construction --- interactivetool_wilson.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interactivetool_wilson.xml b/interactivetool_wilson.xml index c362a758c4e..7f575c43a63 100644 --- a/interactivetool_wilson.xml +++ b/interactivetool_wilson.xml @@ -14,8 +14,8 @@ /home/shiny/.Renviron && - echo $WILSON_BLACKLIST_EXAMPLES >> /home/shiny/.Renviron && + echo "WILSON_LANDING_PAGE=\$WILSON_LANDING_PAGE" > /home/shiny/.Renviron && + echo "WILSON_BLACKLIST_EXAMPLES=\$WILSON_BLACKLIST_EXAMPLES" >> /home/shiny/.Renviron && ln -s ${infile} /srv/shiny-server/external_data/input.clarion && exec shiny-server 2>&1 ]]> From c136c13c5b4bc947ff9dad803c1fd60d818f958f Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Wed, 16 Oct 2019 14:26:04 +0200 Subject: [PATCH 032/324] add wallace IT --- interactivetool_wallace.xml | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 interactivetool_wallace.xml diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml new file mode 100644 index 00000000000..83d70d19f0c --- /dev/null +++ b/interactivetool_wallace.xml @@ -0,0 +1,53 @@ + + Webbased Interactive modeling of species niches and distributions + + quay.io/bgruening/wilson-app + + + + 3838 + /sample-apps/SIG/wallace/shiny/ + + + + ${__app__.security.encode_id($jupyter_notebook.history_id)} + ${__app__.config.galaxy_infrastructure_url} + 8080 + ${__app__.config.galaxy_infrastructure_url} + + #if $__user__: + #for $api_key in $__user__.api_keys: + ${api_key.key} + #break + #end for + #end if + + + + + + + + + + + + + +`_ is a modular platform for reproducible modeling of species niches and distributions. + +.. class:: infomark + + + +]]> + + + 10.1111/2041-210X.12945 + + From fae69abcf39a1833dfda755114627c35b2d6dffa Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Wed, 16 Oct 2019 14:39:58 +0200 Subject: [PATCH 033/324] some updates --- interactivetool_wallace.xml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml index 83d70d19f0c..6a36204caaf 100644 --- a/interactivetool_wallace.xml +++ b/interactivetool_wallace.xml @@ -1,7 +1,7 @@ Webbased Interactive modeling of species niches and distributions - quay.io/bgruening/wilson-app + ylebras/wallace-docker @@ -32,18 +32,30 @@ - + `_ is a modular platform for reproducible modeling of species niches and distributions. .. class:: infomark +Example input file (TAB separated):: + "name" "longitude" "latitude" "countryCode" + Accipiter striatus Vieillot, 1808 -60.291838 46.328137 CA + Accipiter striatus Vieillot, 1808 -114.58927 35.022485 US + Accipiter striatus Vieillot, 1808 -93.37406 30.00586 US + Accipiter striatus Vieillot, 1808 -79.336288 43.682218 CA + Accipiter striatus Vieillot, 1808 -109.156024 31.904185 US + Accipiter striatus Vieillot, 1808 -71.098031 42.297408 US + Accipiter striatus Vieillot, 1808 -110.927215 32.18203 US ]]> From 0d6bc47d2ea2b55eab60f8487978e55635e4f85e Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Wed, 16 Oct 2019 16:34:10 +0200 Subject: [PATCH 034/324] Update interactivetool_wallace.xml --- interactivetool_wallace.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml index 6a36204caaf..66001ee97b9 100644 --- a/interactivetool_wallace.xml +++ b/interactivetool_wallace.xml @@ -29,10 +29,10 @@ ]]> - + - + @@ -42,7 +42,7 @@ -`Wallace `_ is a modular platform for reproducible modeling of species niches and distributions. +`Wallace `_ is a flexible platform for reproducible modeling of species niches and distributions. .. class:: infomark From b6f4fbb1a54657b6d5be4d350f7364652a992c1b Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Wed, 16 Oct 2019 17:05:44 +0200 Subject: [PATCH 035/324] update container input dataset folder After verification --- interactivetool_wallace.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml index 66001ee97b9..5dbf29dfd30 100644 --- a/interactivetool_wallace.xml +++ b/interactivetool_wallace.xml @@ -24,7 +24,7 @@ From b831ce30d1753081d0f95b1bea86cabaa6a86038 Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Wed, 16 Oct 2019 17:35:00 +0200 Subject: [PATCH 036/324] only consider csv occurence file as input for now --- interactivetool_wallace.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml index 5dbf29dfd30..b860dd01267 100644 --- a/interactivetool_wallace.xml +++ b/interactivetool_wallace.xml @@ -29,10 +29,10 @@ ]]> - + - + From f91eb0679ffb88bbe448daa6de4ee5c6c63b2a92 Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Wed, 16 Oct 2019 17:46:12 +0200 Subject: [PATCH 037/324] As we will use Galaxy_helper no need to import data on the fly --- interactivetool_wallace.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml index b860dd01267..cd958ee3d6c 100644 --- a/interactivetool_wallace.xml +++ b/interactivetool_wallace.xml @@ -24,7 +24,6 @@ From 7a75856cd19c83ef5eecf15fce9f7f3af0e5d40c Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Wed, 16 Oct 2019 17:57:14 +0200 Subject: [PATCH 038/324] comment input and output --- interactivetool_wallace.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml index cd958ee3d6c..14b03ac3d3d 100644 --- a/interactivetool_wallace.xml +++ b/interactivetool_wallace.xml @@ -28,10 +28,10 @@ ]]> - + - + From 7bdf4c9c0901924d30effc784c53aa748dd5b78a Mon Sep 17 00:00:00 2001 From: root Date: Wed, 16 Oct 2019 19:29:53 +0200 Subject: [PATCH 039/324] wallace changes --- interactivetool_wallace.xml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml index 14b03ac3d3d..4d8e2c5353b 100644 --- a/interactivetool_wallace.xml +++ b/interactivetool_wallace.xml @@ -4,13 +4,14 @@ ylebras/wallace-docker - + 3838 /sample-apps/SIG/wallace/shiny/ - ${__app__.security.encode_id($jupyter_notebook.history_id)} + + ${__app__.security.encode_id($outfile.history_id)} ${__app__.config.galaxy_infrastructure_url} 8080 ${__app__.config.galaxy_infrastructure_url} @@ -24,14 +25,16 @@ - + From 4599cabec1100eab48286e6caa66d065061d9572 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 23 Oct 2019 19:43:04 +0200 Subject: [PATCH 040/324] update askomics to version 1.0 --- interactivetool_askomics.xml | 56 ++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/interactivetool_askomics.xml b/interactivetool_askomics.xml index b4cb6bff0d1..2dd06fe28ba 100644 --- a/interactivetool_askomics.xml +++ b/interactivetool_askomics.xml @@ -1,16 +1,25 @@ - - AskOmics, a visual SPARQL query builder + + a visual SPARQL query builder - askomics/askomics-ie:19.01.3 + askomics/flaskomics-with-dependencies:3.1.1 - 6543 - /login_api_gie?key=${__user_name__} + 5000 + /loginapikey/${__user_name__} + + true + ${__user_name__} + Galaxy + ${__user_name__} + ${__user_email__} + ${__user_name__} + ${__app__.config.galaxy_infrastructure_url} + #if $__user__: #for $api_key in $__user__.api_keys: @@ -19,27 +28,26 @@ #end for #end if - ${__user_name__} - ${__user_email__} - ${__user_name__} + + prod + 1 + Galaxy + AskOmics Interactive Tool for Galaxy + /tmp/askomics-it + /tmp/askomics-it/database.db + http://localhost:5000 + + true + + 85000 + 65000 - + From 8bbc6f69feda269e8b04208cf9577c4381721954 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 24 Oct 2019 17:12:21 -0400 Subject: [PATCH 041/324] Galaxy InteractiveTools cluster fixes from EU --- interactivetool_jupyter_notebook.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/interactivetool_jupyter_notebook.xml b/interactivetool_jupyter_notebook.xml index 9dd3e36d3ee..60cdc00253f 100644 --- a/interactivetool_jupyter_notebook.xml +++ b/interactivetool_jupyter_notebook.xml @@ -39,7 +39,7 @@ ## copy default notebook cp '$__tool_directory__/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb && jupyter trust ./ipython_galaxy_notebook.ipynb && - jupyter lab --no-browser --NotebookApp.shutdown_button=True && + jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' #else: @@ -50,7 +50,7 @@ #if $mode.run_it: jupyter nbconvert --to notebook --execute --output ./ipython_galaxy_notebook.ipynb --allow-errors ./*.ipynb && #else: - jupyter lab --no-browser --NotebookApp.shutdown_button=True && + jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && #end if cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' #end if @@ -88,13 +88,13 @@ The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations, visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, machine learning, and much more. - + Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. - + You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. - + You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. From a408f3e479f4a17cf53756ddf106fbde009791ad Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 28 Oct 2019 15:58:40 -0400 Subject: [PATCH 042/324] Structured access to Galaxy internals for ITs. --- interactivetool_askomics.xml | 11 ++--------- interactivetool_jupyter_notebook.xml | 15 ++++----------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/interactivetool_askomics.xml b/interactivetool_askomics.xml index c1ce9d3bd11..7f997b54e63 100644 --- a/interactivetool_askomics.xml +++ b/interactivetool_askomics.xml @@ -10,15 +10,8 @@ - ${__app__.config.galaxy_infrastructure_url} - - #if $__user__: - #for $api_key in $__user__.api_keys: - ${api_key.key} - #break - #end for - #end if - + $__galaxy_url + - ${__app__.security.encode_id($jupyter_notebook.history_id)} - ${__app__.config.galaxy_infrastructure_url} + $__history_id + $__galaxy_url 8080 - ${__app__.config.galaxy_infrastructure_url} - - #if $__user__: - #for $api_key in $__user__.api_keys: - ${api_key.key} - #break - #end for - #end if - + $__galaxy_url + Date: Tue, 29 Oct 2019 08:29:02 -0400 Subject: [PATCH 043/324] Cleanup structured IT internals access commit. - Unit test fixes and added tests. - Change __history_id to __history_id__ per comment from @bgruening --- interactivetool_askomics.xml | 2 +- interactivetool_jupyter_notebook.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/interactivetool_askomics.xml b/interactivetool_askomics.xml index 7f997b54e63..b5f0b4d56a4 100644 --- a/interactivetool_askomics.xml +++ b/interactivetool_askomics.xml @@ -10,7 +10,7 @@ - $__galaxy_url + $__galaxy_url__ - $__history_id - $__galaxy_url + $__history_id__ + $__galaxy_url__ 8080 - $__galaxy_url + $__galaxy_url__ Date: Tue, 12 Nov 2019 00:59:32 +0100 Subject: [PATCH 044/324] add pyiron tool --- interactivetool_pyiron.xml | 96 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 interactivetool_pyiron.xml diff --git a/interactivetool_pyiron.xml b/interactivetool_pyiron.xml new file mode 100644 index 00000000000..d56abbb3b66 --- /dev/null +++ b/interactivetool_pyiron.xml @@ -0,0 +1,96 @@ + + + quay.io/bgruening/docker-jupyter-notebook:pyiron + + + + 8888 + ipython/lab + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + pyiron - an integrated development environment (IDE) for computational materials science. It combines several tools in a common platform: + + The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations, + visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, + machine learning, and much more. + + Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to + do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. + + You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. + If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. + + You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. + + From 9eabec14dffb25a8a69bf8d186d26f710fd7db76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 12 Nov 2019 09:37:05 +0100 Subject: [PATCH 045/324] add new envs to pyiron --- interactivetool_pyiron.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/interactivetool_pyiron.xml b/interactivetool_pyiron.xml index d56abbb3b66..0f5ac65a00f 100644 --- a/interactivetool_pyiron.xml +++ b/interactivetool_pyiron.xml @@ -14,10 +14,13 @@ 8080 $__galaxy_url__ + /home/jovyan/resources Date: Tue, 12 Nov 2019 10:48:13 +0100 Subject: [PATCH 046/324] small edits to pyiron --- interactivetool_pyiron.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interactivetool_pyiron.xml b/interactivetool_pyiron.xml index 0f5ac65a00f..9657bc87c3b 100644 --- a/interactivetool_pyiron.xml +++ b/interactivetool_pyiron.xml @@ -3,7 +3,7 @@ quay.io/bgruening/docker-jupyter-notebook:pyiron - + 8888 ipython/lab @@ -70,7 +70,7 @@ - + From 8c752d470358d1fc561764299aa7a40092ba2ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 12 Nov 2019 10:55:57 +0100 Subject: [PATCH 047/324] use proper PWD --- interactivetool_pyiron.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interactivetool_pyiron.xml b/interactivetool_pyiron.xml index 9657bc87c3b..68d59040727 100644 --- a/interactivetool_pyiron.xml +++ b/interactivetool_pyiron.xml @@ -18,8 +18,8 @@ Date: Tue, 12 Nov 2019 13:54:09 +0100 Subject: [PATCH 048/324] Update interactivetool_pyiron.xml --- interactivetool_pyiron.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/interactivetool_pyiron.xml b/interactivetool_pyiron.xml index 68d59040727..69caecdf71b 100644 --- a/interactivetool_pyiron.xml +++ b/interactivetool_pyiron.xml @@ -29,6 +29,7 @@ ## change into the directory where the notebooks are located cd ./jupyter/ && + cp \${HOME}/examples/* ./ && export PATH=/home/jovyan/.local/bin:\$PATH && #if $mode.mode_select == 'scratch': From f953b39beacb6f628a08a6c5fba236891e0c214f Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Wed, 13 Nov 2019 16:29:48 +0100 Subject: [PATCH 049/324] add guacamole_desktop --- interactivetool_guacamole_desktop.xml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 interactivetool_guacamole_desktop.xml diff --git a/interactivetool_guacamole_desktop.xml b/interactivetool_guacamole_desktop.xml new file mode 100644 index 00000000000..a63ce108991 --- /dev/null +++ b/interactivetool_guacamole_desktop.xml @@ -0,0 +1,26 @@ + + + cyverse/ubuntu18-xfce-desktop + + + + 8000 + + + + + + + + + + + + + Simple Ubuntu XFCE all-in-one desktop. + Username: "user" + Password: "password" + + From 99da6e95b01f1051f5393bbb14fc5bb383183a8f Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Thu, 14 Nov 2019 02:16:18 +0100 Subject: [PATCH 050/324] several enhancements and fixes --- interactivetool_guacamole_desktop.xml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/interactivetool_guacamole_desktop.xml b/interactivetool_guacamole_desktop.xml index a63ce108991..665f4d33eb8 100644 --- a/interactivetool_guacamole_desktop.xml +++ b/interactivetool_guacamole_desktop.xml @@ -1,13 +1,15 @@ - cyverse/ubuntu18-xfce-desktop + quay.io/bgruening/guacamole-desktop - 8000 + 8080 + - @@ -19,8 +21,7 @@ - Simple Ubuntu XFCE all-in-one desktop. - Username: "user" - Password: "password" + Simple Ubuntu XFCE all-in-one desktop. The Username is "user" and the Password is "password". + This image is based on the awesome work from CyVerse. From ce872ac1ae789a8770ddf037fc75170a73e35816 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 25 Nov 2019 22:16:04 +0100 Subject: [PATCH 051/324] add new interactive VCF tool --- interactivetool_vcf_iobio.xml | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 interactivetool_vcf_iobio.xml diff --git a/interactivetool_vcf_iobio.xml b/interactivetool_vcf_iobio.xml new file mode 100644 index 00000000000..898a34f7444 --- /dev/null +++ b/interactivetool_vcf_iobio.xml @@ -0,0 +1,54 @@ + + + qiaoy/iobio-bundle.vcf-iobio:dev-ondemand + + + + 80 + + + + /tmp/app.conf && + mv /tmp/app.conf /etc/supervisor.d/app.conf && + + /usr/bin/supervisord -c /etc/supervisord.conf + ]]> + + + + + + + + + + + VCF iobio visualisation. + + From 5ff93313a59871f6cb620924403caae8e6e0afba Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 26 Nov 2019 09:27:22 +0100 Subject: [PATCH 052/324] add climate notebook --- interactivetool_climate_notebook.xml | 95 ++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 interactivetool_climate_notebook.xml diff --git a/interactivetool_climate_notebook.xml b/interactivetool_climate_notebook.xml new file mode 100644 index 00000000000..1d12e46316e --- /dev/null +++ b/interactivetool_climate_notebook.xml @@ -0,0 +1,95 @@ + + + nordicesmhub/docker-climate-notebook:1.0 + + + + 8888 + ipython/lab + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The Climate Notebook is based on Jupyter an open-source web application that allows you to create and share documents that contain live code, equations, + visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, + machine learning, and much more. + + Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to + do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. + + You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. + If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. + + You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. + + The Climate version of the Jupyter Notebook offers a lot of preinstalled tools for climate science. + + From 03dc5828f9a402ce23675a9de841d921b147afe5 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Tue, 26 Nov 2019 13:30:13 +0100 Subject: [PATCH 053/324] complement help for interactive climate notebook by adding the list of available packages and a link to pangeo --- interactivetool_climate_notebook.xml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/interactivetool_climate_notebook.xml b/interactivetool_climate_notebook.xml index 1d12e46316e..aa1692ada80 100644 --- a/interactivetool_climate_notebook.xml +++ b/interactivetool_climate_notebook.xml @@ -90,6 +90,22 @@ You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. - The Climate version of the Jupyter Notebook offers a lot of preinstalled tools for climate science. + The Climate version of the Jupyter Notebook offers a lot of preinstalled tools for climate science. The list of packages is based on what is available + on the [Pangeo](http://pangeo.io/) platform and [Pangeo stacks](https://pangeo-data.github.io/pangeo-stacks/). + + - **Core scipy packages**: numpy, scipy, matplotlib, pandas, xarray, sparse and sympy + - **Data science**: scikit-image, scikit-learn, dask-ml, tensorflow, keras, pytorch-cpu, dask_labextension + - **Visualization**: holoviews, panel, geoviews, hvplot, geoviews, datashader, seaborn, altair, descartes, folium, vega, + vega_datasets, palettable, cmocean,plotly, psy-maps, psy-reg, psyplot, psyplot-gui, psy-maps, psy-reg, + geopy, branca + - **Geospatial**: iris, cartopy, basemap, basemap-data-hires, geopandas, rasterio, netcdf4, erddapy, pydap, h5py, h5netcdf, regionmask and rio-cogeo + - **Geoscience related**: climlab, metpy, satpy, gsw, eofs, esmpy, xesmf, windspharm, rasterstats, geojsoncontour + - **Climate related**: pyaerocom, cdo, cdsapi, cfgrib, cis. esmvaltool, nc-time-axis, nco + - **Intake related**: intake, intake-xarray, intake-esm, fsspec and intake-stac + - **zarr related**: zarr, numcodecs, python-blosc, lz4, gcsfs, s3fs, tiledb-py + - **jupyter related**: ipyleaflet, papermill, jupytext, ipydatawidgets, sidecar + - **xarray related**: xgcm, xrft, xhistogram, xlrd, xrviz, climpred, pytide, pyinterp + - **misc**: python-wget, prefect, requests, pillow, pip, nbgitpuller, pysplit, biopython, bioblend and galaxy-ie-helpers + From 91d2eea4ae54ef5f3f03000757dddef135d76290 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 26 Nov 2019 17:13:14 +0100 Subject: [PATCH 054/324] add some VCF fixes --- interactivetool_vcf_iobio.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/interactivetool_vcf_iobio.xml b/interactivetool_vcf_iobio.xml index 898a34f7444..4bb1d4eeec9 100644 --- a/interactivetool_vcf_iobio.xml +++ b/interactivetool_vcf_iobio.xml @@ -16,12 +16,12 @@ #set $PUB_HTTP_PORT = '80' cd /var/www/html && - ##sed -i "s@\"wss://services.iobio.io/samtools/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/samtools/\"@" js/bam.iobio.js/bam.iobio.js && - ##sed -i "s@\"wss://services.iobio.io/bamreaddepther/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/bamreaddepther/\"@" js/bam.iobio.js/bam.iobio.js && - sed -i "s@\"wss://services.iobio.io/vcfreaddepther/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/vcfreaddepther/\"@" js/vcf.iobio.js/vcf.iobio.js && - sed -i "s@\"wss://services.iobio.io/vcfstatsalive/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/vcfstatsalive/\"@" js/vcf.iobio.js/vcf.iobio.js && - ##sed -i "s@\"wss://services.iobio.io/bamstatsalive/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/bamstatsalive/\"@" js/bam.iobio.js/bam.iobio.js && - sed -i "s@\"wss://services.iobio.io/tabix/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/tabix/\"@" js/vcf.iobio.js/vcf.iobio.js && + ##sed -i "s@\"wss://services.iobio.io/samtools/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/samtools/\"@" app/bam.iobio.js && + ##sed -i "s@\"wss://services.iobio.io/bamreaddepther/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/bamreaddepther/\"@" app/bam.iobio.js && + sed -i "s@\"wss://services.iobio.io/vcfdepther/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/vcfdepther/\"@" app/vcf.iobio.js && + sed -i "s@\"wss://services.iobio.io/vcfstatsalive/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/vcfstatsalive/\"@" app/vcf.iobio.js && + ##sed -i "s@\"wss://services.iobio.io/bamstatsalive/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/bamstatsalive/\"@" app/bam.iobio.js && + sed -i "s@\"wss://services.iobio.io/tabix/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/tabix/\"@" app/vcf.iobio.js && ##s@ws://tabix.iobio.io@ws://" + window.location.hostname + ":8000@ From da4d1a043c47f38c4228e397b9847c55c62fb20c Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 26 Nov 2019 19:58:33 +0100 Subject: [PATCH 055/324] update vcf IE --- interactivetool_vcf_iobio.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/interactivetool_vcf_iobio.xml b/interactivetool_vcf_iobio.xml index 4bb1d4eeec9..dc585d8818c 100644 --- a/interactivetool_vcf_iobio.xml +++ b/interactivetool_vcf_iobio.xml @@ -5,7 +5,7 @@ 80 - + /tmp/app.conf && mv /tmp/app.conf /etc/supervisor.d/app.conf && From eb84e6743d81ad1fc602085f5c10b8500ad82486 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Wed, 27 Nov 2019 13:14:19 +0100 Subject: [PATCH 056/324] add latest VCF changes --- interactivetool_vcf_iobio.xml | 39 ++++++++++++++++------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/interactivetool_vcf_iobio.xml b/interactivetool_vcf_iobio.xml index dc585d8818c..3be0266c593 100644 --- a/interactivetool_vcf_iobio.xml +++ b/interactivetool_vcf_iobio.xml @@ -5,49 +5,46 @@ 80 - + /tmp/app.conf && + mv /tmp/app.conf /etc/supervisor.d/app.conf && + /usr/bin/supervisord -c /etc/supervisord.conf - sed -i 's/deny all;//g' /etc/nginx/nginx.conf && - - cp '${infile}' /input/vcffile.vcf && - ##cp '${infile.metadata.bam_index}' /input/vcffile.bam.bai && - head -n -2 /etc/supervisor.d/app.conf > /tmp/app.conf && - mv /tmp/app.conf /etc/supervisor.d/app.conf && - - /usr/bin/supervisord -c /etc/supervisord.conf ]]> - + - - VCF iobio visualisation. + `_. + +This visualization is using Galaxy Interactive Tool and utilizes an all-in-one Docker container from http://iobio.io. + +Make sure your VCF file is compressed to the vcf_bgzip datatype to load it into the Visualization. + ]]> + From a6ed2d61ba622850748963327b6f63c4ffbfa72a Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Sat, 30 Nov 2019 22:46:10 +0100 Subject: [PATCH 057/324] A first try of openrefine GIE -> interactivetool --- interactivetool_openrefine.xml | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 interactivetool_openrefine.xml diff --git a/interactivetool_openrefine.xml b/interactivetool_openrefine.xml new file mode 100644 index 00000000000..17680fb6d0e --- /dev/null +++ b/interactivetool_openrefine.xml @@ -0,0 +1,64 @@ + + Working with messy data: cleaning it; transforming it from one format into another; and extending it with web services and external data. + + ylebras/openrefine-docker + + + + 80 + + + + + + ${__app__.security.encode_id($outfile.history_id)} + ${__app__.config.galaxy_infrastructure_url} + 8080 + ${__app__.config.galaxy_infrastructure_url} + + #if $__user__: + #for $api_key in $__user__.api_keys: + ${api_key.key} + #break + #end for + #end if + + + + + + + + + + + + + +`_ (previously Google Refine) is a powerful tool for working with messy data: cleaning it; transforming it from one format into another; and extending it with web services and external data. + +.. class:: infomark + +Example input file (TAB separated):: + + "name" "longitude" "latitude" "countryCode" + Accipiter striatus Vieillot, 1808 -60.291838 46.328137 CA + Accipiter striatus Vieillot, 1808 -114.58927 35.022485 US + Accipiter striatus Vieillot, 1808 -93.37406 30.00586 US + Accipiter striatus Vieillot, 1808 -79.336288 43.682218 CA + Accipiter striatus Vieillot, 1808 -109.156024 31.904185 US + Accipiter striatus Vieillot, 1808 -71.098031 42.297408 US + Accipiter striatus Vieillot, 1808 -110.927215 32.18203 US + +]]> + + From a97099d7cced23d10f4833a2c2f9ff7ce331924f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 1 Dec 2019 14:17:01 +0100 Subject: [PATCH 058/324] so this should work, I hope --- interactivetool_openrefine.xml | 63 ++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/interactivetool_openrefine.xml b/interactivetool_openrefine.xml index 17680fb6d0e..956073db48a 100644 --- a/interactivetool_openrefine.xml +++ b/interactivetool_openrefine.xml @@ -1,32 +1,46 @@ - - Working with messy data: cleaning it; transforming it from one format into another; and extending it with web services and external data. + + Working with messy data ylebras/openrefine-docker - 80 - + 3333 - - ${__app__.security.encode_id($outfile.history_id)} - ${__app__.config.galaxy_infrastructure_url} + $__history_id__ + $__galaxy_url__ 8080 - ${__app__.config.galaxy_infrastructure_url} - - #if $__user__: - #for $api_key in $__user__.api_keys: - ${api_key.key} - #break - #end for - #end if - + $__galaxy_url__ + - + \&1) +while [[ \${STATUS} =~ "refused" ]] +do + echo "Waiting for openrefine: \$STATUS \n" + STATUS=\$(curl --include 'http://127.0.0.1:3333' 2>\&1) + sleep 4 +done +]]> + + + @@ -34,17 +48,16 @@ - + + + - -`_ (previously Google Refine) is a powerful tool for working with messy data: cleaning it; transforming it from one format into another; and extending it with web services and external data. +`Openrefine `_ (previously Google Refine) is a powerful tool for working with messy data: +cleaning it; transforming it from one format into another; and extending it with web services and external data. .. class:: infomark From 466d8d38ab48a829205e17a37d38f581aa90db6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 1 Dec 2019 14:34:43 +0100 Subject: [PATCH 059/324] small fix --- interactivetool_openrefine.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interactivetool_openrefine.xml b/interactivetool_openrefine.xml index 956073db48a..11a0b01ffc3 100644 --- a/interactivetool_openrefine.xml +++ b/interactivetool_openrefine.xml @@ -20,11 +20,11 @@ exec /OpenRefine/refine -i 0.0.0.0 -m \$GALAXY_MEMORY_MB & ##Check if openrefine is up to work -STATUS=\$(curl --include 'http://127.0.0.1:3333' 2>\&1) +STATUS=\$(curl --include 'http://127.0.0.1:3333' 2>&1) while [[ \${STATUS} =~ "refused" ]] do echo "Waiting for openrefine: \$STATUS \n" - STATUS=\$(curl --include 'http://127.0.0.1:3333' 2>\&1) + STATUS=\$(curl --include 'http://127.0.0.1:3333' 2>&1) sleep 4 done ]]> From 7fcbf9c6dd142d91c6f5cabd6d067f3be4e81c76 Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Mon, 2 Dec 2019 11:34:43 +0100 Subject: [PATCH 060/324] Create interactivetool_radiant.xml First try --- interactivetool_radiant.xml | 75 +++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 interactivetool_radiant.xml diff --git a/interactivetool_radiant.xml b/interactivetool_radiant.xml new file mode 100644 index 00000000000..d48cd9cf126 --- /dev/null +++ b/interactivetool_radiant.xml @@ -0,0 +1,75 @@ + + Data analytics using Radiant R Shiny app + + ylebras/radiant-docker + + + + 3838 + /sample-apps/STAT/ + + + + + ${__app__.security.encode_id($outfile.history_id)} + ${__app__.config.galaxy_infrastructure_url} + 8080 + ${__app__.config.galaxy_infrastructure_url} + + #if $__user__: + #for $api_key in $__user__.api_keys: + ${api_key.key} + #break + #end for + #end if + + + + + + + + + + + + + +`_ is an open-source platform-independent browser-based interface for business analytics in R. The application is based on the Shiny package and can be run locally or on a server. Radiant was developed by Vincent Nijs. Please use the issue tracker on GitHub to suggest enhancements or report problems: https://github.com/radiant-rstats/radiant/issues. For other questions and comments please use radiant@rady.ucsd.edu. +Key features + +- Explore: Quickly and easily summarize, visualize, and analyze your data +- Cross-platform: It runs in a browser on Windows, Mac, and Linux +- Reproducible: Recreate results and share work with others as a state file or an Rmarkdown report +- Programming: Integrate Radiant’s analysis functions with your own R-code +- Context: Data and examples focus on business applications + + +.. class:: infomark + +Example input file (TAB separated):: + + "name" "longitude" "latitude" "countryCode" + Accipiter striatus Vieillot, 1808 -60.291838 46.328137 CA + Accipiter striatus Vieillot, 1808 -114.58927 35.022485 US + Accipiter striatus Vieillot, 1808 -93.37406 30.00586 US + Accipiter striatus Vieillot, 1808 -79.336288 43.682218 CA + Accipiter striatus Vieillot, 1808 -109.156024 31.904185 US + Accipiter striatus Vieillot, 1808 -71.098031 42.297408 US + Accipiter striatus Vieillot, 1808 -110.927215 32.18203 US + +]]> + + + + + From b9df1d6eb01024381abbf0d5f7b87646829e1bca Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Mon, 2 Dec 2019 11:47:00 +0100 Subject: [PATCH 061/324] Create interactivetool_geoexplorer.xml --- interactivetool_geoexplorer.xml | 69 +++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 interactivetool_geoexplorer.xml diff --git a/interactivetool_geoexplorer.xml b/interactivetool_geoexplorer.xml new file mode 100644 index 00000000000..ba51c7c70ed --- /dev/null +++ b/interactivetool_geoexplorer.xml @@ -0,0 +1,69 @@ + + An interactive spatial analysis platform using ggvis and Leaflet + + ylebras/geoexplorer-docker + + + + 3838 + /sample-apps/SIG/ + + + + + ${__app__.security.encode_id($outfile.history_id)} + ${__app__.config.galaxy_infrastructure_url} + 8080 + ${__app__.config.galaxy_infrastructure_url} + + #if $__user__: + #for $api_key in $__user__.api_keys: + ${api_key.key} + #break + #end for + #end if + + + + + + + + + + + + + +`_ is An interactive spatial analysis platform using ggvis and Leaflet. + +Author: David Stephens + +App: http://www.davesteps.com/geoExploreR/ + +.. class:: infomark + +Example input file (TAB separated):: + + "name" "longitude" "latitude" "countryCode" + Accipiter striatus Vieillot, 1808 -60.291838 46.328137 CA + Accipiter striatus Vieillot, 1808 -114.58927 35.022485 US + Accipiter striatus Vieillot, 1808 -93.37406 30.00586 US + Accipiter striatus Vieillot, 1808 -79.336288 43.682218 CA + Accipiter striatus Vieillot, 1808 -109.156024 31.904185 US + Accipiter striatus Vieillot, 1808 -71.098031 42.297408 US + Accipiter striatus Vieillot, 1808 -110.927215 32.18203 US + +]]> + + + + + From ea43b3c5b5a5b62a12b820d50712a93006231df5 Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Mon, 2 Dec 2019 13:56:23 +0100 Subject: [PATCH 062/324] correct Galaxy environment variables --- interactivetool_geoexplorer.xml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/interactivetool_geoexplorer.xml b/interactivetool_geoexplorer.xml index ba51c7c70ed..0667c42d5d7 100644 --- a/interactivetool_geoexplorer.xml +++ b/interactivetool_geoexplorer.xml @@ -10,19 +10,12 @@ - - ${__app__.security.encode_id($outfile.history_id)} - ${__app__.config.galaxy_infrastructure_url} + $__history_id__ + $__galaxy_url__ 8080 - ${__app__.config.galaxy_infrastructure_url} - - #if $__user__: - #for $api_key in $__user__.api_keys: - ${api_key.key} - #break - #end for - #end if - + $__galaxy_url__ + + Date: Mon, 2 Dec 2019 13:57:32 +0100 Subject: [PATCH 063/324] correct env variables --- interactivetool_radiant.xml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/interactivetool_radiant.xml b/interactivetool_radiant.xml index d48cd9cf126..7b87bfe3ead 100644 --- a/interactivetool_radiant.xml +++ b/interactivetool_radiant.xml @@ -10,19 +10,12 @@ - - ${__app__.security.encode_id($outfile.history_id)} - ${__app__.config.galaxy_infrastructure_url} + $__history_id__ + $__galaxy_url__ 8080 - ${__app__.config.galaxy_infrastructure_url} - - #if $__user__: - #for $api_key in $__user__.api_keys: - ${api_key.key} - #break - #end for - #end if - + $__galaxy_url__ + + Date: Mon, 2 Dec 2019 14:10:39 +0100 Subject: [PATCH 064/324] update citation --- interactivetool_geoexplorer.xml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/interactivetool_geoexplorer.xml b/interactivetool_geoexplorer.xml index 0667c42d5d7..894dd590c23 100644 --- a/interactivetool_geoexplorer.xml +++ b/interactivetool_geoexplorer.xml @@ -57,6 +57,12 @@ Example input file (TAB separated):: ]]> - + @misc{githubsurvey2018, + author = {davesteps}, + title = {{dashboard to visualise geographic data}}, + publisher = {Github}, + url = {https://github.com/davesteps/geoExploreR} + } + } From 980b7344a1d3019e22ff64b438383da654dab4fd Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Mon, 2 Dec 2019 14:12:40 +0100 Subject: [PATCH 065/324] update citation --- interactivetool_radiant.xml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/interactivetool_radiant.xml b/interactivetool_radiant.xml index 7b87bfe3ead..ef7165b9897 100644 --- a/interactivetool_radiant.xml +++ b/interactivetool_radiant.xml @@ -63,6 +63,12 @@ Example input file (TAB separated):: ]]> - + @misc{githubsurvey2018, + author = {vnijs}, + title = {{Radiant - Business analytics using R and Shiny}}, + publisher = {Github}, + url = {https://github.com/vnijs/radiant} + } + } From c4b5f3ff41e958b3f06897dd4e4c22897b8b7d6a Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Mon, 2 Dec 2019 14:15:14 +0100 Subject: [PATCH 066/324] remove empty lines --- interactivetool_radiant.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/interactivetool_radiant.xml b/interactivetool_radiant.xml index ef7165b9897..40615e71a0a 100644 --- a/interactivetool_radiant.xml +++ b/interactivetool_radiant.xml @@ -18,9 +18,7 @@ @@ -33,10 +31,6 @@ `_ is an open-source platform-independent browser-based interface for business analytics in R. The application is based on the Shiny package and can be run locally or on a server. Radiant was developed by Vincent Nijs. Please use the issue tracker on GitHub to suggest enhancements or report problems: https://github.com/radiant-rstats/radiant/issues. For other questions and comments please use radiant@rady.ucsd.edu. Key features @@ -46,7 +40,6 @@ Key features - Programming: Integrate Radiant’s analysis functions with your own R-code - Context: Data and examples focus on business applications - .. class:: infomark Example input file (TAB separated):: From b4bc34122677563dcd6f8c328b584fef3890ec27 Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Mon, 2 Dec 2019 17:30:55 +0100 Subject: [PATCH 067/324] modify url --- interactivetool_radiant.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_radiant.xml b/interactivetool_radiant.xml index 40615e71a0a..691a6ec1ae9 100644 --- a/interactivetool_radiant.xml +++ b/interactivetool_radiant.xml @@ -6,7 +6,7 @@ 3838 - /sample-apps/STAT/ + /sample-apps/STAT/inst/app From b5bcaead979c8cc04d24bb77a95cc30c1bb19597 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 2 Dec 2019 20:15:58 +0100 Subject: [PATCH 068/324] small changes to new ITs --- interactivetool_geoexplorer.xml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/interactivetool_geoexplorer.xml b/interactivetool_geoexplorer.xml index 894dd590c23..d709f0c0360 100644 --- a/interactivetool_geoexplorer.xml +++ b/interactivetool_geoexplorer.xml @@ -15,19 +15,23 @@ 8080 $__galaxy_url__ - > /var/log/shiny-server.log 2>&1 ]]> - + - + From 53bc67882fef3797bfacc9608d6a953098fe98ad Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 2 Dec 2019 20:16:03 +0100 Subject: [PATCH 069/324] small changes to new ITs --- interactivetool_radiant.xml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/interactivetool_radiant.xml b/interactivetool_radiant.xml index 40615e71a0a..ba84b9f4372 100644 --- a/interactivetool_radiant.xml +++ b/interactivetool_radiant.xml @@ -15,17 +15,21 @@ 8080 $__galaxy_url__ - > /var/log/shiny-server.log 2>&1 ]]> - + From 70d84c7d933a062ea0049394b24cf40b54315806 Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Tue, 3 Dec 2019 09:09:42 +0100 Subject: [PATCH 070/324] add input file mandatory format --- interactivetool_geoexplorer.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/interactivetool_geoexplorer.xml b/interactivetool_geoexplorer.xml index 894dd590c23..c5420332887 100644 --- a/interactivetool_geoexplorer.xml +++ b/interactivetool_geoexplorer.xml @@ -43,9 +43,11 @@ App: http://www.davesteps.com/geoExploreR/ .. class:: infomark +**Input data file MUST have as longitude column clomun 2, and as latitude column, column 3** + Example input file (TAB separated):: - "name" "longitude" "latitude" "countryCode" + "name" "x" "y" "countryCode" Accipiter striatus Vieillot, 1808 -60.291838 46.328137 CA Accipiter striatus Vieillot, 1808 -114.58927 35.022485 US Accipiter striatus Vieillot, 1808 -93.37406 30.00586 US From 63c7d543b0c9cc187dc9bfde09172c8fa8e474fb Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 13 Dec 2019 09:49:19 +0100 Subject: [PATCH 071/324] Fix env var name + update --- interactivetool_askomics.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interactivetool_askomics.xml b/interactivetool_askomics.xml index 095d27f9f2b..94cf111a2f3 100644 --- a/interactivetool_askomics.xml +++ b/interactivetool_askomics.xml @@ -1,7 +1,7 @@ a visual SPARQL query builder - askomics/flaskomics-with-dependencies:3.1.1 + askomics/flaskomics-with-dependencies:3.2.0 @@ -19,7 +19,7 @@ ${__user_name__} $__galaxy_url__ - + prod 1 From 1d78a090d8ea0ca2ecbd94f759534698b9c83cdf Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Fri, 13 Dec 2019 17:39:57 +0100 Subject: [PATCH 072/324] add data import (#43) I was frogetting the data import part apparently :) --- interactivetool_radiant.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/interactivetool_radiant.xml b/interactivetool_radiant.xml index 769dc90f5cc..67c0205cc0e 100644 --- a/interactivetool_radiant.xml +++ b/interactivetool_radiant.xml @@ -17,6 +17,10 @@ Date: Fri, 13 Dec 2019 19:14:33 +0100 Subject: [PATCH 073/324] Copy inputdata instead of ln -s + help section update (#44) * Copy inputdata instead of ln -s * Copy inputdata instead of ln -s * Modify input datatype in help --- interactivetool_geoexplorer.xml | 18 +++++++++--------- interactivetool_radiant.xml | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/interactivetool_geoexplorer.xml b/interactivetool_geoexplorer.xml index 77375fd5725..7256cb402c2 100644 --- a/interactivetool_geoexplorer.xml +++ b/interactivetool_geoexplorer.xml @@ -19,7 +19,7 @@ diff --git a/interactivetool_radiant.xml b/interactivetool_radiant.xml index 67c0205cc0e..05b623d0ffc 100644 --- a/interactivetool_radiant.xml +++ b/interactivetool_radiant.xml @@ -19,7 +19,7 @@ Date: Tue, 17 Dec 2019 00:01:25 +0100 Subject: [PATCH 074/324] Update interactivetool_jupyter_notebook.xml --- interactivetool_jupyter_notebook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_jupyter_notebook.xml b/interactivetool_jupyter_notebook.xml index 4665d48a3c3..8ecafd6f75a 100644 --- a/interactivetool_jupyter_notebook.xml +++ b/interactivetool_jupyter_notebook.xml @@ -22,7 +22,7 @@ mkdir -p ./jupyter/data && #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) - ln -sf '$input' './jupyter/data/${cleaned_name}' && + cp '$input' './jupyter/data/${cleaned_name}' && ## change into the directory where the notebooks are located cd ./jupyter/ && From bcf600696722b41a79db12cd9aed8edbffdc036a Mon Sep 17 00:00:00 2001 From: Yvan Le Bras Date: Fri, 20 Dec 2019 19:38:16 +0100 Subject: [PATCH 075/324] Update help and input data file type (#45) --- interactivetool_geoexplorer.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/interactivetool_geoexplorer.xml b/interactivetool_geoexplorer.xml index 7256cb402c2..3c8e1f5a536 100644 --- a/interactivetool_geoexplorer.xml +++ b/interactivetool_geoexplorer.xml @@ -47,18 +47,18 @@ App: http://www.davesteps.com/geoExploreR/ .. class:: infomark -**Input data file MUST have as longitude column clomun 2, and as latitude column, column 3** +**Input data file MUST have as a uniq ID per row on first column, longitude column on column 2, latitude column on column 3 and quantitative values on 4th column ** Example input file (csv):: - "name" "x" "y" "countryCode" - "Accipiter striatus Vieillot, 1808" -60.291838 46.328137 CA - "Accipiter striatus Vieillot, 1808" -114.58927 35.022485 US - "Accipiter striatus Vieillot, 1808" -93.37406 30.00586 US - "Accipiter striatus Vieillot, 1808" -79.336288 43.682218 CA - "Accipiter striatus Vieillot, 1808" -109.156024 31.904185 US - "Accipiter striatus Vieillot, 1808" -71.098031 42.297408 US - "Accipiter striatus Vieillot, 1808" -110.927215 32.18203 US +"ID" "x" "y" "test" +01 -60.291838 46.328137 2 +02 -114.58927 35.022485 3 +03 -93.37406 30.00586 4 +04 -79.336288 43.682218 5 +05 -109.156024 31.904185 2 +06 -71.098031 42.297408 9 +07 -110.927215 32.18203 12 ]]> From a94abfefa779a42d555350a93ca8129c5fd19f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E7=BB=B4=E7=87=95=28Steven=29?= Date: Mon, 13 Jan 2020 11:41:02 +0800 Subject: [PATCH 076/324] Fix sentting email error in python-3.7 Fix sentting email error( `ValueError: server_hostname cannot be an empty string or start with a leading dot.`) in python-3.7. --- lib/galaxy/util/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 1dcb20b2303..70c1e2fd44d 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1481,9 +1481,9 @@ def send_mail(frm, to, subject, body, config, html=None): smtp_ssl = asbool(getattr(config, 'smtp_ssl', False)) if smtp_ssl: - s = smtplib.SMTP_SSL() + s = smtplib.SMTP_SSL(config.smtp_server) else: - s = smtplib.SMTP() + s = smtplib.SMTP(config.smtp_server) s.connect(config.smtp_server) if not smtp_ssl: try: From 4d461a5e91d24bf8d4c28aeaa95a166d765138d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 16 Jan 2020 19:28:56 +0100 Subject: [PATCH 077/324] fix jupyter --- interactivetool_jupyter_notebook.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/interactivetool_jupyter_notebook.xml b/interactivetool_jupyter_notebook.xml index 8ecafd6f75a..c86717460df 100644 --- a/interactivetool_jupyter_notebook.xml +++ b/interactivetool_jupyter_notebook.xml @@ -21,9 +21,10 @@ mkdir -p ./jupyter/outputs/ && mkdir -p ./jupyter/data && - #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) - cp '$input' './jupyter/data/${cleaned_name}' && - + #if $input: + #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) + cp '$input' './jupyter/data/${cleaned_name}' && + #end if ## change into the directory where the notebooks are located cd ./jupyter/ && export PATH=/home/jovyan/.local/bin:\$PATH && From 944ee259d07b060513928c131d6f94481e44cbc4 Mon Sep 17 00:00:00 2001 From: Joachim Wolff Date: Thu, 16 Jan 2020 20:58:36 +0100 Subject: [PATCH 078/324] Release 19.09 europe (#47) * Interactive tool for higlass * Changes as requested --- interactivetool_higlass.xml | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 interactivetool_higlass.xml diff --git a/interactivetool_higlass.xml b/interactivetool_higlass.xml new file mode 100644 index 00000000000..8f3231e28c4 --- /dev/null +++ b/interactivetool_higlass.xml @@ -0,0 +1,38 @@ + + an interactive Hi-C data visualizer + + higlass/higlass-docker + + + + 80 + + + + + + + + + + + + + + Interactive tool for visualising Hi-C data, works only for multi-cooler files which store multiple resolutions. + For a detailed documentaition please visit https://docs.higlass.io/. + + + + 10.1186/s13059-018-1486-1 + + + From 40640440185384998a915b7059059f3b3779ada6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 16 Jan 2020 21:44:08 +0100 Subject: [PATCH 079/324] restrict tool to mcool file --- interactivetool_higlass.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_higlass.xml b/interactivetool_higlass.xml index 8f3231e28c4..1571b811a3a 100644 --- a/interactivetool_higlass.xml +++ b/interactivetool_higlass.xml @@ -19,7 +19,7 @@ ]]> - + From 4229b1783ebee7d2228770edd30ba0d131dab0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Jan 2020 09:22:08 +0100 Subject: [PATCH 080/324] Update interactivetool_cellxgene.xml --- interactivetool_cellxgene.xml | 95 +++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/interactivetool_cellxgene.xml b/interactivetool_cellxgene.xml index 546bd7ab58f..3c99f3a393b 100644 --- a/interactivetool_cellxgene.xml +++ b/interactivetool_cellxgene.xml @@ -3,27 +3,112 @@ quay.io/galaxy/cellxgene-galaxy-ie:ie2 - + 80 - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + experimental_annotations['experimental_annotations_select'] == 'enable' + - Interactive tool for visualising AnnData. + An interactive explorer for single-cell transcriptomics (AnnData formatted) data + + cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the Human Cell Atlas. Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data. + + Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data. + https://github.com/chanzuckerberg/cellxgene + + 10.5281/zenodo.3554576 + From 7dc87aed78de749e2226049a503627c36d644d7c Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 31 Jan 2020 13:38:36 +0100 Subject: [PATCH 081/324] data_column use_header_names test and docs - add a test (also as example) - clarify in the docs --- lib/galaxy/tool_util/xsd/galaxy.xsd | 8 +++++--- test/functional/tools/column_param.xml | 10 ++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 6ae0e3e102c..f994bc85272 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2563,8 +2563,10 @@ Set to ``false`` to not force user to select an option in the list.Used only if the ``type`` attribute value is ``data_column``, if ``true`` Galaxy assumes first row of ``data_ref`` is a header and builds the select list with these values rather than the more -generic ``c1`` ... ``cN``. - +generic ``c1`` ... ``cN`` (i.e. it will be ``c1: head1`` ... ``cN: headN``). +Note that the content of the Cheetah variable is still +the column index. + @@ -4830,7 +4832,7 @@ write out a JSON representation of the tool parameters. *Example* -The following will create a cheetah variable that can be evaluated as ``$inputs`` that +The following will create a Cheetah variable that can be evaluated as ``$inputs`` that will contain the tool parameter inputs. ```xml diff --git a/test/functional/tools/column_param.xml b/test/functional/tools/column_param.xml index 92b979210e7..f89979179df 100644 --- a/test/functional/tools/column_param.xml +++ b/test/functional/tools/column_param.xml @@ -1,10 +1,13 @@ - + '$output1' - + ]]> + @@ -13,8 +16,11 @@ + + + From b5b4f0955627408205509eeded6ff818f1db602b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 31 Jan 2020 13:41:49 +0100 Subject: [PATCH 082/324] fix potential bug empty columns must not be stripped --- lib/galaxy/tools/parameters/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 5fc67474e97..b6cecb959f5 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1271,7 +1271,7 @@ class ColumnListParameter(SelectToolParameter): try: with open(dataset.get_file_name(), 'r') as f: head = f.readline() - cnames = head.rstrip().split('\t') + cnames = head.rstrip("\n\r ").split('\t') column_list = [('%d' % (i + 1), 'c%d: %s' % (i + 1, x)) for i, x in enumerate(cnames)] if self.numerical: # If numerical was requested, filter columns based on metadata if hasattr(dataset, 'metadata') and hasattr(dataset.metadata, 'column_types'): From 61e227dd855db95cee23d4e15015a91b606dee2e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 31 Jan 2020 15:53:32 +0100 Subject: [PATCH 083/324] add missing tool output --- test/functional/tools/column_param.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/tools/column_param.xml b/test/functional/tools/column_param.xml index f89979179df..dad77b65b6b 100644 --- a/test/functional/tools/column_param.xml +++ b/test/functional/tools/column_param.xml @@ -1,8 +1,8 @@ '$output1' + echo "col $col" >> '$output1' && + echo "col_names $col_names" >> '$output1' && + cut -f '$col' '$input1' >> '$output1' ]]> From a93a69dc5932f6686248d24e70334ad38b073ec5 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sun, 2 Feb 2020 13:44:05 +0100 Subject: [PATCH 084/324] add panoply as new interative tool (#50) --- interactivetool_panoply.xml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 interactivetool_panoply.xml diff --git a/interactivetool_panoply.xml b/interactivetool_panoply.xml new file mode 100644 index 00000000000..6abb7b8e2c9 --- /dev/null +++ b/interactivetool_panoply.xml @@ -0,0 +1,28 @@ + + + quay.io/nordicesmhub/docker-panoply + + + + 5800 + + + + + + + + + + + + + + `Panoply `_ plots geo-referenced and other arrays from netCDF, HDF, GRIB, and other datasets. + + From 081dd5ec7307fec1bb1805ad3b77eee634415bb5 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 2 Feb 2020 23:24:45 +0100 Subject: [PATCH 085/324] fix panoply IT --- interactivetool_panoply.xml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/interactivetool_panoply.xml b/interactivetool_panoply.xml index 6abb7b8e2c9..66c18d5139f 100644 --- a/interactivetool_panoply.xml +++ b/interactivetool_panoply.xml @@ -1,4 +1,5 @@ - + + interative plotting tool for geo-referenced data quay.io/nordicesmhub/docker-panoply @@ -9,9 +10,9 @@ @@ -22,7 +23,8 @@ - + `_ plots geo-referenced and other arrays from netCDF, HDF, GRIB, and other datasets. + ]]> From f04a60a9b46055894b30a21212220f61dcca436c Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 3 Feb 2020 08:00:16 +0100 Subject: [PATCH 086/324] higlass changes --- interactivetool_higlass.xml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/interactivetool_higlass.xml b/interactivetool_higlass.xml index 1571b811a3a..20ffda8df0b 100644 --- a/interactivetool_higlass.xml +++ b/interactivetool_higlass.xml @@ -1,7 +1,7 @@ an interactive Hi-C data visualizer - higlass/higlass-docker + image-default @@ -9,13 +9,12 @@ From 85a441148f606c30242d9a614f63cf4a1ac20c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Mon, 3 Feb 2020 23:30:14 +0100 Subject: [PATCH 087/324] Update interactivetool_higlass.xml --- interactivetool_higlass.xml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/interactivetool_higlass.xml b/interactivetool_higlass.xml index 20ffda8df0b..6d0510925da 100644 --- a/interactivetool_higlass.xml +++ b/interactivetool_higlass.xml @@ -1,7 +1,7 @@ an interactive Hi-C data visualizer - image-default + quay.io/bgruening/galaxy-higlass @@ -9,19 +9,20 @@ - + - + @@ -35,3 +36,4 @@ + From 4c7c2da3bd7aa6d5c38c8cb70beea90a302db4ca Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sat, 8 Feb 2020 20:26:29 +0100 Subject: [PATCH 088/324] try to fix panoply interactive tool (#51) * try to fix panoply interactive tool * add colorbars for panoply * remove colorbars moved colorbars to dockerfile. Untar colorbars.tar in the user home --- interactivetool_panoply.xml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/interactivetool_panoply.xml b/interactivetool_panoply.xml index 66c18d5139f..2323bcae5c3 100644 --- a/interactivetool_panoply.xml +++ b/interactivetool_panoply.xml @@ -9,17 +9,26 @@ - + + + From 55dc0e6a2f361dbcbeda6fec22be38da27e17fd2 Mon Sep 17 00:00:00 2001 From: Gianmauro Cuccuru Date: Tue, 11 Feb 2020 11:16:37 +0100 Subject: [PATCH 089/324] add a new function to get the local ip address --- lib/galaxy_ext/container_monitor/monitor.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/galaxy_ext/container_monitor/monitor.py b/lib/galaxy_ext/container_monitor/monitor.py index 86987b81e2d..35e3c58d6fd 100644 --- a/lib/galaxy_ext/container_monitor/monitor.py +++ b/lib/galaxy_ext/container_monitor/monitor.py @@ -12,6 +12,19 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa from galaxy.tool_util.deps import docker_util +def get_ip(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + # doesn't even have to be reachable + s.connect(('10.255.255.255', 1)) + ip = s.getsockname()[0] + except Exception: + ip = None + finally: + s.close() + return ip + + def parse_ports(container_name, connection_configuration): while True: ports_command = docker_util.build_docker_simple_command("port", container_name=container_name, **connection_configuration) @@ -41,11 +54,7 @@ def main(): try: ports_raw = parse_ports(container_name, connection_configuration) if ports_raw is not None: - try: - host_ip = socket.gethostbyname(socket.gethostname()) - except Exception: - # doesn't work on OS X - host_ip = None + host_ip = get_ip() with open("container_runtime.json", "w") as f: ports = docker_util.parse_port_text(ports_raw) if host_ip is not None: From e5d1a7f1670c48e2077c9a2abc15eabdf352007e Mon Sep 17 00:00:00 2001 From: Gianmauro Cuccuru Date: Tue, 11 Feb 2020 15:07:07 +0100 Subject: [PATCH 090/324] move the function into a different path --- lib/galaxy/util/sockets.py | 13 +++++++++++++ lib/galaxy_ext/container_monitor/monitor.py | 15 +-------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/util/sockets.py b/lib/galaxy/util/sockets.py index ffc246627e1..826e69c1a23 100644 --- a/lib/galaxy/util/sockets.py +++ b/lib/galaxy/util/sockets.py @@ -4,6 +4,19 @@ import socket import subprocess +def get_ip(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + # doesn't even have to be reachable + s.connect(('10.255.255.255', 1)) + ip = s.getsockname()[0] + except Exception: + ip = None + finally: + s.close() + return ip + + def unused_port(range=None): if range: return __unused_port_on_range(range) diff --git a/lib/galaxy_ext/container_monitor/monitor.py b/lib/galaxy_ext/container_monitor/monitor.py index 35e3c58d6fd..e3254f2cc70 100644 --- a/lib/galaxy_ext/container_monitor/monitor.py +++ b/lib/galaxy_ext/container_monitor/monitor.py @@ -1,6 +1,5 @@ import json import os -import socket import subprocess import sys import tempfile @@ -10,19 +9,7 @@ import time sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) from galaxy.tool_util.deps import docker_util - - -def get_ip(): - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # doesn't even have to be reachable - s.connect(('10.255.255.255', 1)) - ip = s.getsockname()[0] - except Exception: - ip = None - finally: - s.close() - return ip +from galaxy.util.sockets import get_ip def parse_ports(container_name, connection_configuration): From bda4454671564810febbe4491a923e95d47c6344 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sat, 15 Feb 2020 12:22:19 +0100 Subject: [PATCH 091/324] update docker for interactive climate (#53) * update docker for interactive climate * test if data or not before copy --- interactivetool_climate_notebook.xml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/interactivetool_climate_notebook.xml b/interactivetool_climate_notebook.xml index aa1692ada80..0ddca7eacb8 100644 --- a/interactivetool_climate_notebook.xml +++ b/interactivetool_climate_notebook.xml @@ -1,6 +1,6 @@ - + - nordicesmhub/docker-climate-notebook:1.0 + nordicesmhub/docker-climate-notebook:1.2 @@ -21,8 +21,10 @@ mkdir -p ./jupyter/outputs/ && mkdir -p ./jupyter/data && - #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) - ln -sf '$input' './jupyter/data/${cleaned_name}' && + #if $input: + #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) + cp '$input' './jupyter/data/${cleaned_name}' && + #end if ## change into the directory where the notebooks are located cd ./jupyter/ && From 79d63abf72566c85dfdd486e587005796e78f42f Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sat, 15 Feb 2020 15:01:26 +0100 Subject: [PATCH 092/324] Try to follow panoply developer recommendations (#52) * Try to follow panoply developer recommendations to fix errors * bug fix: do not fail if no png If users do not create any png files, it should not fail --- interactivetool_panoply.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/interactivetool_panoply.xml b/interactivetool_panoply.xml index 2323bcae5c3..5e84abffe8a 100644 --- a/interactivetool_panoply.xml +++ b/interactivetool_panoply.xml @@ -9,17 +9,17 @@ From bc1013c001e1207bbf1b24334db97d350b1c1d07 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 22 Dec 2019 11:03:55 +0100 Subject: [PATCH 093/324] Add more repository installation test cases These were supposed to not pass without https://github.com/galaxyproject/galaxy/pull/9159, but they do. I don't think this increases test coverage but it might be handy to have this as integration test cases when we rip out the toolshed from the Galaxy code base ? --- test/integration/test_repository_operations.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/integration/test_repository_operations.py b/test/integration/test_repository_operations.py index ea41c27ae22..13a8c010372 100644 --- a/test/integration/test_repository_operations.py +++ b/test/integration/test_repository_operations.py @@ -41,6 +41,21 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC self._install_repository() self._uninstall_repository() + def test_datatype_uninstall(self): + repo = ('devteam', 'blast_datatypes', '01b38f20197e') + self.install_repository(*repo) + self.uninstall_repository(*repo) + + def test_package_uninstall(self): + repo = ('iuc', 'package_fastqc_0_11_4', 'a8f485b2efd9') + self.install_repository(*repo) + self.uninstall_repository(*repo) + + def test_tool_with_package_dependency_uninstall(self): + repo = ('iuc', 'fastqc', 'e7b2202befea') + self.install_repository(*repo) + self.uninstall_repository(*repo) + def test_repository_update(self): response = self._install_repository(revision=REVISION_4, version="0.0.3")[0] assert response['ctx_rev'] == '4' From e84784d9a7852de36abb12397734e4754b2d25c2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 20 Feb 2020 14:50:03 -0500 Subject: [PATCH 094/324] Disable the admin_toolshed controller (except for tests). --- .../galaxy/controllers/admin_toolshed.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index eb0126d8b7a..3c5824172b7 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1,12 +1,14 @@ import json import logging import os +from functools import wraps from six import string_types from sqlalchemy import or_ import tool_shed.repository_types.util as rt_util from galaxy import util, web +from galaxy.exceptions import ConfigDoesNotAllowException from galaxy.tool_shed.galaxy_install import install_manager from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager @@ -31,12 +33,24 @@ from .admin import AdminGalaxy log = logging.getLogger(__name__) +def legacy_tool_shed_endpoint(func): + # admin only and only available if running test cases. + @wraps(func) + def wrapper(trans, *args, **kwargs): + if not trans.app.config.config_dict.get("running_functional_tests", False): + raise ConfigDoesNotAllowException("Legacy tool shed endpoint only available during testing.") + return func(trans, *args, **kwargs) + + return wrapper + + class AdminToolshed(AdminGalaxy): installed_repository_grid = admin_toolshed_grids.InstalledRepositoryGrid() @web.expose @web.require_admin + @legacy_tool_shed_endpoint def activate_repository(self, trans, **kwd): """Activate a repository that was deactivated but not uninstalled.""" repository_id = kwd['id'] @@ -59,6 +73,7 @@ class AdminToolshed(AdminGalaxy): @web.legacy_expose_api @web.require_admin + @legacy_tool_shed_endpoint def browse_repositories(self, trans, **kwd): message = kwd.get('message', '') status = kwd.get('status', '') @@ -73,6 +88,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def restore_repository(self, trans, **kwd): repository_id = kwd['id'] repository = repository_util.get_installed_tool_shed_repository(trans.app, repository_id) @@ -157,6 +173,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def view_tool_metadata(self, trans, repository_id, tool_id, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') @@ -193,11 +210,13 @@ class AdminToolshed(AdminGalaxy): @web.json @web.require_admin @web.do_not_cache + @legacy_tool_shed_endpoint def get_file_contents(self, trans, file_path, repository_id): return suc.get_repository_file_contents(trans.app, file_path, repository_id, is_admin=True) @web.expose @web.require_admin + @legacy_tool_shed_endpoint def get_tool_dependencies(self, trans, repository_id, repository_name, repository_owner, changeset_revision): """ Send a request to the appropriate tool shed to retrieve the dictionary of tool dependencies defined for @@ -224,6 +243,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def get_updated_repository_information(self, trans, repository_id, repository_name, repository_owner, changeset_revision): """ Send a request to the appropriate tool shed to retrieve the dictionary of information required to reinstall @@ -246,6 +266,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def initiate_tool_dependency_installation(self, trans, tool_dependencies, **kwd): """ Install specified dependencies for repository tools. The received list of tool_dependencies @@ -285,6 +306,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def install_latest_repository_revision(self, trans, **kwd): """Install the latest installable revision of a repository that has been previously installed.""" repository_id = kwd.get('id', None) @@ -336,6 +358,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def install_tool_dependencies_with_update(self, trans, **kwd): """ Updating an installed tool shed repository where new tool dependencies but no new repository @@ -411,6 +434,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def install_repositories(self, trans, **kwd): reinstalling = util.string_as_bool(kwd.get('reinstalling', False)) encoded_kwd = kwd.get('encoded_kwd') @@ -436,6 +460,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def manage_repository(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') @@ -509,6 +534,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def monitor_repository_installation(self, trans, **kwd): tsr_ids = common_util.get_tool_shed_repository_ids(**kwd) if not tsr_ids: @@ -530,6 +556,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def prepare_for_install(self, trans, **kwd): if not suc.have_shed_tool_conf_for_install(trans.app): message = 'The tool_config_file setting in galaxy.ini must include at least one ' @@ -810,6 +837,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def reinstall_repository(self, trans, **kwd): """ Reinstall a tool shed repository that has been previously uninstalled, making sure to handle all repository @@ -981,6 +1009,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def reselect_tool_panel_section(self, trans, **kwd): """ Select or change the tool panel section to contain the tools included in the tool shed repository @@ -1151,6 +1180,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def uninstall_tool_dependencies(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') @@ -1195,6 +1225,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def update_to_changeset_revision(self, trans, **kwd): """Update a cloned repository to the latest revision possible.""" message = escape(kwd.get('message', '')) From e8c5c05a509022c380724995a1187acff58a641a Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 15 Nov 2019 15:06:16 +0100 Subject: [PATCH 095/324] datatypes: new data types for OpenMS - added MascotXML (datatype and sniffer) - added ConsensusXML, IdXML, FeatureXML, Ms2, Wiff - sorted the sniffers alphabetically to simplify maintenance needed here galaxyproteomics/tools-galaxyp#266 --- .../config/sample/datatypes_conf.xml.sample | 29 ++++++++++++------- lib/galaxy/datatypes/proteomics.py | 7 +++++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index f2aacad6d03..a5727cf4729 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -248,6 +248,7 @@ + @@ -835,21 +836,27 @@ - - - - + + + + - - - - - - - + + + + + + + + + + + + + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index ee6c4bc8d7b..c651e8161fb 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -127,6 +127,13 @@ class PepXml(ProteomicsXml): root = "msms_pipeline_analysis" +class MascotXML(ProteomicsXml): + """mzXML data""" + file_ext = "mascotxml" + blurb = "mascot Mass Spectrometry data" + root = "mascot_search_results" + + class MzML(ProteomicsXml): """mzML data""" edam_format = "format_3244" From d9613233147759ca50f1db59825df4c19783f49e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 18 Nov 2019 23:39:24 +0100 Subject: [PATCH 096/324] add XQuest XML data type --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/proteomics.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index a5727cf4729..a1641789086 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -262,6 +262,7 @@ + @@ -856,6 +857,7 @@ + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index c651e8161fb..6d36c6456fb 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -225,6 +225,12 @@ class UniProtXML(ProteomicsXml): root = "uniprot" +class XquestXML(ProteomicsXml): + file_ext = "xquest.xml" + blurb = "XQuest XML file" + root = "xquest_results" + + class Mgf(Text): """Mascot Generic Format data""" edam_data = "data_2536" From 43c8281d95a7203846edd4eaaefec5c3ed9a612d Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 19 Nov 2019 12:20:39 +0100 Subject: [PATCH 097/324] add pqp file format subject to https://github.com/OpenMS/OpenMS/issues/4365 --- .../config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/binary.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index a1641789086..ad5745085ce 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -258,6 +258,7 @@ + @@ -790,6 +791,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 36b655d30c3..cbfef66ee19 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1570,6 +1570,25 @@ class MzSQlite(SQlite): return False +class PQP(SQlite): + """Class describing a Peptide query parameters file""" + file_ext = "pqp" + + def set_meta(self, dataset, overwrite=True, **kwd): + super(PQP, self).set_meta(dataset, overwrite=overwrite, **kwd) + + def sniff(self, filename): + """ + table definition according to https://github.com/grosenberger/OpenMS/blob/develop/src/openms/source/ANALYSIS/OPENSWATH/TransitionPQPFile.cpp#L264 + for now VERSION GENE PEPTIDE_GENE_MAPPING are excluded, since + there is test data wo these tables, see also here https://github.com/OpenMS/OpenMS/issues/4365 + """ + table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', + 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', + 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING'] + return super(PQP, self).sniff(filename) and self.sniff_table_names(filename, table_names) + + class BlibSQlite(SQlite): """Class describing a Proteomics Spectral Library Sqlite database """ MetadataElement(name="blib_version", default='1.8', param=MetadataParameter, desc="Blib Version", From 9da6e8e16952a4c0857ce92d3d52e8dfdca491f7 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sat, 22 Feb 2020 15:21:59 +0100 Subject: [PATCH 098/324] use get to copy data/notebook for jupyterlab (#54) ITs --- interactivetool_climate_notebook.xml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/interactivetool_climate_notebook.xml b/interactivetool_climate_notebook.xml index 0ddca7eacb8..f867208bfbe 100644 --- a/interactivetool_climate_notebook.xml +++ b/interactivetool_climate_notebook.xml @@ -23,7 +23,8 @@ #if $input: #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) - cp '$input' './jupyter/data/${cleaned_name}' && + get -t hid -i '${input.hid}' && + ln -sf '/import/${input.hid}' './jupyter/data/${cleaned_name}' && #end if ## change into the directory where the notebooks are located @@ -32,15 +33,16 @@ #if $mode.mode_select == 'scratch': ## copy default notebook - cp '$__tool_directory__/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb && + cp '/home/jovyan/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb && jupyter trust ./ipython_galaxy_notebook.ipynb && jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' #else: - #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) - cp '$mode.ipynb' ./${cleaned_name}.ipynb && - jupyter trust ./${cleaned_name}.ipynb && + #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($mode.ipynb.element_identifier)) + get -t hid -i '${mode.ipynb.hid}' && + ln -sf '/import/${mode.ipynb.hid}' './jupyter/data/${cleaned_name}' && + jupyter trust ./${cleaned_name} && #if $mode.run_it: jupyter nbconvert --to notebook --execute --output ./ipython_galaxy_notebook.ipynb --allow-errors ./*.ipynb && From 69828b837a97d068ab78182f788507e0b35753e1 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 21 Nov 2019 10:16:10 +0100 Subject: [PATCH 099/324] added osw data type --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/binary.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index ad5745085ce..09538a00eb8 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -258,6 +258,7 @@ + @@ -791,6 +792,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index cbfef66ee19..7520f0e3686 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1589,6 +1589,20 @@ class PQP(SQlite): return super(PQP, self).sniff(filename) and self.sniff_table_names(filename, table_names) +class OSW(PQP): + """Class describing OpenSwath output""" + file_ext = "osw" + + def set_meta(self, dataset, overwrite=True, **kwd): + super(OSW, self).set_meta(dataset, overwrite=overwrite, **kwd) + + def sniff(self, filename): + # osw seems to be an extension of pqp (few tables are added) + # see also here https://github.com/OpenMS/OpenMS/issues/4365 + table_names = ['FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] + return super(OSW, self).sniff(filename) and self.sniff_table_names(filename, table_names) + + class BlibSQlite(SQlite): """Class describing a Proteomics Spectral Library Sqlite database """ MetadataElement(name="blib_version", default='1.8', param=MetadataParameter, desc="Blib Version", From a6b743d85d3183041b4060636b6a961f2fe2c18d Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 20 Nov 2019 22:38:54 +0100 Subject: [PATCH 100/324] added mzTab(2) and TarfoXML --- .../config/sample/datatypes_conf.xml.sample | 7 +- lib/galaxy/datatypes/proteomics.py | 68 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 09538a00eb8..e89381ea88a 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -233,7 +233,6 @@ - @@ -249,6 +248,8 @@ + + @@ -261,6 +262,7 @@ + @@ -852,6 +854,8 @@ + + @@ -860,6 +864,7 @@ + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 6d36c6456fb..bc4f5cbcc6a 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -53,6 +53,68 @@ class Wiff(Binary): return "\n".join(rval) +@build_sniff_from_prefix +class MzTab(Text): + """exchange format for proteomics and metabolomics results""" + edam_data = "data_3681" + file_ext = "mztab" + # section names (except MTD) + __sections = ["PRH", "PRT", "PEH", "PEP", "PSH", "PSM", "SMH", "SML", "COM"] + __version_re = r"([0-9]{1,2})(\.[0-9])?(\.[0-9])?$" + + def __init__(self, **kwd): + super(MzTab, self).__init__(**kwd) + + def set_peek(self, dataset, is_multi_byte=False): + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = data.get_file_peek(dataset.file_name) + dataset.blurb = 'mzTab Format' + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def sniff_prefix(self, file_prefix): + """ Determines whether the file is the correct type. """ + has_version = False + has_mode = False + has_type = False + + contents = file_prefix.string_io() + while True: + line = contents.readline().split("\t") + if line[0] == "MTD": + if line[1] == "mzTab-version" and re.match(self.__version_re, line) is not None: + has_version = True + elif line[1] == "mzTab-mode" and line[2].lowercase() in ["complete", "summary"]: + has_mode = True + elif line[1] == "mzTab-type" and line[2].lowercase() in ['quantification', 'identification']: + has_type = True + elif not line[0] in self.__sections: + return False + + return has_version and has_mode and has_type + + +class MzTab2(MzTab): + """exchange format for proteomics and metabolomics results""" + file_ext = "mztab2" + __sections = ["SMH", "SML", "SFH", "SMF", "SEH", "SME", "COM"] + __version_re = r"([0-9]{1,2})(\.[0-9])?(\.[0-9])?-M$" + + def __init__(self, **kwd): + super(MzTab2, self).__init__(**kwd) + + def set_peek(self, dataset, is_multi_byte=False): + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = data.get_file_peek(dataset.file_name) + dataset.blurb = 'mzTab2 Format' + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + class PepXmlReport(Tabular): """pepxml converted to tabular report""" edam_data = "data_2536" @@ -187,6 +249,12 @@ class TraML(ProteomicsXml): root = "TraML" +class TrafoXML(ProteomicsXml): + file_ext = "trafoxml" + blurb = "RT alignment tranformation" + root = "TrafoXML" + + class MzQuantML(ProteomicsXml): edam_format = "format_3248" file_ext = "mzq" From acc7aad6011a451525bfd6d874806c79e13b94e8 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 20 Nov 2019 23:04:10 +0100 Subject: [PATCH 101/324] strip spaces from lines for detecting proteomics xml files to allow for files like https://github.com/OpenMS/OpenMS/blob/develop/src/tests/topp/FileFilter_44_output.mzML --- lib/galaxy/datatypes/proteomics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index bc4f5cbcc6a..572746bebdc 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -164,7 +164,7 @@ class ProteomicsXml(GenericXml): """ Determines whether the file is the correct XML type. """ contents = file_prefix.string_io() while True: - line = contents.readline() + line = contents.readline().strip() if line is None or not line.startswith(' Date: Tue, 26 Nov 2019 13:31:19 +0100 Subject: [PATCH 102/324] add sqmass --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/binary.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index e89381ea88a..dd24f068f81 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -350,6 +350,7 @@ + @@ -793,6 +794,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 7520f0e3686..9a48239111a 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1603,6 +1603,20 @@ class OSW(PQP): return super(OSW, self).sniff(filename) and self.sniff_table_names(filename, table_names) +class SQmass(SQlite): + """Class describing a Sqmass database """ + file_ext = "sqmass" + + def set_meta(self, dataset, overwrite=True, **kwd): + super(SQmass, self).set_meta(dataset, overwrite=overwrite, **kwd) + + def sniff(self, filename): + if super(SQmass, self).sniff(filename): + table_names = ["CHROMATOGRAM", "PRECURSOR", "RUN", "SPECTRUM", "DATA", "PRODUCT", "RUN_EXTRA"] + return self.sniff_table_names(filename, table_names) + return False + + class BlibSQlite(SQlite): """Class describing a Proteomics Spectral Library Sqlite database """ MetadataElement(name="blib_version", default='1.8', param=MetadataParameter, desc="Blib Version", From b9dda087a97a8d70f4cd483293c6205a7dd4ae67 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 28 Nov 2019 09:52:47 +0100 Subject: [PATCH 103/324] add Kroenik --- .../config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/proteomics.py | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index dd24f068f81..07c222e7800 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -234,6 +234,7 @@ + @@ -860,6 +861,7 @@ + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 572746bebdc..4b1649a0020 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -115,6 +115,33 @@ class MzTab2(MzTab): dataset.blurb = 'file purged from disk' +@build_sniff_from_prefix +class Kroenik(Tabular): + """ Kroenik (HardKloer sibling) files""" + file_ext = "kroenik" + + def __init__(self, **kwd): + super(Kroenik, self).__init__(**kwd) + self.column_names = ["File", "First Scan", "Last Scan", "Num of Scans", "Charge", "Monoisotopic Mass", "Base Isotope Peak", "Best Intensity", "Summed Intensity", "First RTime", "Last RTime", "Best RTime", "Best Correlation", "Modifications"] + + def display_peek(self, dataset): + """Returns formated html of peek""" + return self.make_html_table(dataset, column_names=self.column_names) + + def sniff_prefix(self, file_prefix): + fh = file_prefix.string_io() + line = [_.strip() for _ in fh.readline().split("\t")] + if len(line) != len(self.column_names) or line != self.column_names: + return False + line = fh.readline().split("\t") + try: + [int(_) for _ in line[1:5]] + [float(_) for _ in line[5:13]] + except ValueError: + return False + return True + + class PepXmlReport(Tabular): """pepxml converted to tabular report""" edam_data = "data_2536" From 80a7737f71069dc605081130b9c44d8d00552745 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 29 Nov 2019 14:55:25 +0100 Subject: [PATCH 104/324] add dta, dta2d, edta --- .../config/sample/datatypes_conf.xml.sample | 6 + lib/galaxy/datatypes/proteomics.py | 273 +++++++++++++++++- 2 files changed, 278 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 07c222e7800..f65b7b989c3 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -228,6 +228,9 @@ + + + @@ -846,6 +849,9 @@ + + + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 4b1649a0020..85e86fda437 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -8,7 +8,7 @@ from galaxy.datatypes import data from galaxy.datatypes.binary import Binary from galaxy.datatypes.data import Text from galaxy.datatypes.sniff import build_sniff_from_prefix -from galaxy.datatypes.tabular import Tabular +from galaxy.datatypes.tabular import Tabular, TabularData from galaxy.datatypes.xml import GenericXml from galaxy.util import nice_size @@ -181,6 +181,277 @@ class ProtXmlReport(Tabular): return self.make_html_table(dataset, column_names=self.column_names) +@build_sniff_from_prefix +class Dta(TabularData): + """dta + The first line contains the singly protonated peptide mass (MH+) and the + peptide charge state separated by a space. Subsequent lines contain space + separated pairs of fragment ion m/z and intensity values. + """ + file_ext = "dta" + comment_lines = 0 + + def set_meta(self, dataset, **kwd): + column_types = [] + data_row = [] + data_lines = 0 + if dataset.has_data(): + with open(dataset.file_name, 'r') as dtafile: + for line in dtafile: + data_lines += 1 + + # Guess column types + for cell in data_row: + column_types.append(self.guess_type(cell)) + + # Set metadata + dataset.metadata.data_lines = data_lines + dataset.metadata.comment_lines = 0 + dataset.metadata.column_types = ['float', 'float'] + dataset.metadata.columns = 2 + dataset.metadata.column_names = ['m/z', 'intensity'] + dataset.metadata.delimiter = " " + + def sniff_prefix(self, file_prefix): + has_data = False + for line in file_prefix.line_iterator(): + line = line.strip().split(" ") + if len(line) != 2: + return False + try: + line = [float(_) for _ in line] + except ValueError: + return False + if not all(_ >= 0 for _ in line): + return False + has_data = True + return has_data + + +@build_sniff_from_prefix +class Dta2d(TabularData): + """ + dta2d: files with three tab/space-separated columns. + The default format is: retention time (seconds) , m/z , intensity. + If the first line starts with '#', a different order is defined by the the + order of the keywords 'MIN' (retention time in minutes) or 'SEC' (retention + time in seconds), 'MZ', and 'INT'. + Example: '#MZ MIN INT' + The peaks of one retention time have to be in subsequent lines. + + Note: sniffer detects + - tab separated with correct header and + - all space separated variants w/wo correct header + Since tab separated wo header would probably cover to much + """ + file_ext = "dta2d" + comment_lines = 0 + + def _parse_header(self, line): + if len(line) != 3 or len(line[0]) < 3 or not line[0].startswith("#"): + return None + line[0] = line[0].lstrip("#") + line = [_.strip() for _ in line] + if 'MZ' not in line or 'INT' not in line or ('MIN' not in line and 'SEC' not in line): + return None + return line + + def _parse_delimiter(self, line): + if len(line.split(" ")) == 3: + return " " + elif len(line.split("\t")) == 3: + return "\t" + return None + + def _parse_dataline(self, line): + try: + line = [float(_) for _ in line] + except ValueError: + return False + if not all(_ >= 0 for _ in line): + return False + return True + + def set_meta(self, dataset, **kwd): + data_lines = 0 + delim = None + if dataset.has_data(): + with open(dataset.file_name, 'r') as dtafile: + for line in dtafile: + if delim is None: + delim = self._parse_delimiter(line) + dataset.metadata.column_names = self._parse_header(line.split(delim)) + data_lines += 1 + + # Set metadata + if delim is not None: + dataset.metadata.delimiter = delim + + dataset.metadata.data_lines = data_lines + dataset.metadata.comment_lines = 0 + dataset.metadata.column_types = ['float', 'float', 'float'] + dataset.metadata.columns = 3 + if dataset.metadata.column_names is None or dataset.metadata.column_names == []: + dataset.metadata.comment_lines += 1 + dataset.metadata.data_lines -= 1 + dataset.metadata.column_names = ['SEC', 'MZ', 'INT'] + + def sniff_prefix(self, file_prefix): + sep = None + header = None + for idx, line in enumerate(file_prefix.line_iterator()): + line = line.strip() + if sep is None: + sep = self._parse_delimiter(line) + if sep is None: + return False + line = line.split(sep) + if len(line) != 3: + return False + if idx == 0: + header = self._parse_header(line) + if (header is None) and not self._parse_dataline(line): + return False + elif not self._parse_dataline(line): + return False + if sep is None or (sep == '\t' and header is None): + return False + return True + + +@build_sniff_from_prefix +class Edta(TabularData): + """ + Input text file containing tab, space or comma separated columns. + The separator between columns is checked in the first line in this order. + + It supports three variants of this format. + + 1. Columns are: RT, MZ, Intensity A header is optional. + 2. Columns are: RT, MZ, Intensity, Charge, columns{0,} A header is mandatory. + 3. Columns are: (RT, MZ, Intensity, Charge){1,}, columns{0,} + Header is mandatory. First quadruplet is the consensus. All following + quadruplets describe the sub-features. This variant is discerned from + variant #2 by the name of the fifth column, which is required to be RT1 + (or rt1). All other column names for sub-features are faithfully ignored. + """ + file_ext = "edta" + comment_lines = 0 + + def _parse_delimiter(self, line): + if len(line.split(" ")) >= 3: + return " " + elif len(line.split("\t")) >= 3: + return "\t" + elif len(line.split(",")) >= 3: + return "\t" + return None + + def _parse_type(self, line): + """ + parse the type from the header line + types 1-3 as in the class docs, 0: type 1 wo/wrong header + """ + if len(line) < 3: + return None + line = [_.lower().replace("/", "") for _ in line] + if len(line) == 3: + if line[0] == "rt" and line[1] == "mz" and (line[2] == "int" or line[2] == "intensity"): + return 1 + else: + return 0 + if line[0] != "rt" or line[1] != "mz" or (line[2] != "int" and line[2] != "intensity") or line[3] != "charge": + return None + if not line[4].startswith("rt"): + return 2 + else: + return 3 + + def _parse_dataline(self, line, tpe): + if tpe == 2 or tpe == 3: + l = 4 + else: + l = 3 + try: + line = [float(_) for _ in line[:l]] + except ValueError: + return False + if not all(_ >= 0 for _ in line[:l]): + return False + return True + + def _clean_header(self, line): + for idx, el in enumerate(line): + el = el.lower() + if el.startswith("rt"): + line[idx] = "RT" + elif el.startswith("int"): + line[idx] = "intensity" + elif el.startswith("mz"): + line[idx] = "m/z" + elif el.startswith("charge"): + line[idx] = "charge" + else: + break + if idx // 4 > 0: + line[idx] += str(idx // 4) + return line + + def set_meta(self, dataset, **kwd): + data_lines = 0 + delim = None + if dataset.has_data(): + with open(dataset.file_name, 'r') as dtafile: + for idx, line in enumerate(dtafile): + if idx == 0: + delim = self._parse_delimiter(line) + tpe = self._parse_type(line.split(delim)) + if tpe == 0: + dataset.metadata.column_names = ["RT", "m/z", "intensity"] + else: + dataset.metadata.column_names = self._clean_header(line.split(delim)) + data_lines += 1 + + # Set metadata + if delim is not None: + dataset.metadata.delimiter = delim + for c in dataset.metadata.column_names: + if any(c.startswith(_) for _ in ["RT", "m/z", "intensity", "charge"]): + dataset.metadata.column_types.append("float") + else: + dataset.metadata.column_types.append("str") + + dataset.metadata.data_lines = data_lines + dataset.metadata.comment_lines = 0 + dataset.metadata.columns = len(dataset.metadata.column_names) + if tpe > 0: + dataset.metadata.comment_lines += 1 + dataset.metadata.data_lines -= 1 + + def sniff_prefix(self, file_prefix): + sep = None + tpe = None + for idx, line in enumerate(file_prefix.line_iterator()): + line = line.strip("\r\n") + if sep is None: + sep = self._parse_delimiter(line) + if sep is None: + return False + line = line.split(sep) + + if idx == 0: + tpe = self._parse_type(line) + if tpe is None: + return False + elif tpe == 0 and not self._parse_dataline(line, tpe): + return False + elif not self._parse_dataline(line, tpe): + return False + if tpe is None or (tpe == 0 and sep == '\t'): + return False + return True + + class ProteomicsXml(GenericXml): """ An enhanced XML datatype used to reuse code across several proteomic/mass-spec datatypes. """ From a285b48f54c44aa5908deaba65af8480269beb25 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 14 Jan 2020 15:02:24 +0100 Subject: [PATCH 105/324] dta2d, edta: remove sniffing of to generic types of these files --- lib/galaxy/datatypes/proteomics.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 85e86fda437..30c0b8bb2bb 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -239,10 +239,8 @@ class Dta2d(TabularData): Example: '#MZ MIN INT' The peaks of one retention time have to be in subsequent lines. - Note: sniffer detects - - tab separated with correct header and - - all space separated variants w/wo correct header - Since tab separated wo header would probably cover to much + Note: sniffer detects (tab or space separated) dta2d files with correct + header, wo header seems to generic """ file_ext = "dta2d" comment_lines = 0 @@ -314,7 +312,7 @@ class Dta2d(TabularData): return False elif not self._parse_dataline(line): return False - if sep is None or (sep == '\t' and header is None): + if sep is None or header is None: return False return True @@ -334,6 +332,8 @@ class Edta(TabularData): quadruplets describe the sub-features. This variant is discerned from variant #2 by the name of the fifth column, which is required to be RT1 (or rt1). All other column names for sub-features are faithfully ignored. + + Note the sniffer only detects files with header. """ file_ext = "edta" comment_lines = 0 @@ -359,7 +359,7 @@ class Edta(TabularData): if line[0] == "rt" and line[1] == "mz" and (line[2] == "int" or line[2] == "intensity"): return 1 else: - return 0 + return None if line[0] != "rt" or line[1] != "mz" or (line[2] != "int" and line[2] != "intensity") or line[3] != "charge": return None if not line[4].startswith("rt"): @@ -447,7 +447,7 @@ class Edta(TabularData): return False elif not self._parse_dataline(line, tpe): return False - if tpe is None or (tpe == 0 and sep == '\t'): + if tpe is None: return False return True From 69644fbfabc7fdd23c5b4c2427b0c8ac078ad2df Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 6 Dec 2019 16:26:51 +0100 Subject: [PATCH 106/324] add paramXML data type --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/proteomics.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index f65b7b989c3..5f9502d875a 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -236,6 +236,7 @@ + @@ -865,6 +866,7 @@ + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 30c0b8bb2bb..b4e5a8b4826 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -479,6 +479,13 @@ class ProteomicsXml(GenericXml): dataset.blurb = 'file purged from disk' +class ParamXml(ProteomicsXml): + """store Parameters in XML formal""" + file_ext = "paramxml" + blurb = "parameters in xmls" + root = "parameters|PARAMETERS" + + class PepXml(ProteomicsXml): """pepXML data""" edam_format = "format_3655" From 4c7cd6d1e8b055c5f77065f714969064285d3c3f Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 9 Dec 2019 10:55:10 +0100 Subject: [PATCH 107/324] add XQuest Spectra data type And search root tag not only at the beginning of the line, i.e. and switch pattern '^<(\w*:)?ROOT' to '<(\w*:)?ROOT' and search instead of match ("^" in combination with match was not necessary anyway). --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/proteomics.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 5f9502d875a..70c58d4e056 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -272,6 +272,7 @@ + @@ -879,6 +880,7 @@ + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index b4e5a8b4826..4aa56df4568 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -466,8 +466,8 @@ class ProteomicsXml(GenericXml): if line is None or not line.startswith(' Date: Mon, 9 Dec 2019 11:25:23 +0100 Subject: [PATCH 108/324] add qcML --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/proteomics.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 70c58d4e056..ea3e55af445 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -273,6 +273,7 @@ + @@ -881,6 +882,7 @@ + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 4aa56df4568..828dc0d462d 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -611,6 +611,16 @@ class XquestSpecXML(ProteomicsXml): root = "xquest_spectra" +class QCML(ProteomicsXml): + """qcml + https://github.com/OpenMS/OpenMS/blob/develop/share/OpenMS/SCHEMAS/mzQCML_0_0_5.xsd + https://github.com/OpenMS/OpenMS/blob/develop/share/OpenMS/SCHEMAS/qcML_0.0.7.xsd + """ + file_ext = "qcml" + blurb = 'QualityAssessments to runs' + root = "qcML|MzQualityML)" + + class Mgf(Text): """Mascot Generic Format data""" edam_data = "data_2536" From aa8d901b3f1a43d674666492cb5635460ca4e7ce Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 18 Dec 2019 12:21:39 +0100 Subject: [PATCH 109/324] add psms and peplist --- .../config/sample/datatypes_conf.xml.sample | 4 ++ lib/galaxy/datatypes/proteomics.py | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index ea3e55af445..5cab3114fc3 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -239,6 +239,8 @@ + + @@ -872,6 +874,8 @@ + + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 828dc0d462d..536bfc202a4 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -142,6 +142,58 @@ class Kroenik(Tabular): return True +@build_sniff_from_prefix +class PepList(Tabular): + """ + Peplist file as used in OpenMS + https://github.com/OpenMS/OpenMS/blob/0fc8765670a0ad625c883f328de60f738f7325a4/src/openms/source/FORMAT/FileHandler.cpp#L432 + """ + file_ext = "peplist" + + def __init__(self, **kwd): + super(PepList, self).__init__(**kwd) + self.column_names = ["m/z", "rt(min)", "snr", "charge", "intensity"] + + def display_peek(self, dataset): + """Returns formated html of peek""" + return self.make_html_table(dataset, column_names=self.column_names) + + def sniff_prefix(self, file_prefix): + fh = file_prefix.string_io() + line = [_.strip() for _ in fh.readline().split("\t")] + if len(line) == 5 and line == self.column_names: + return True + return False + + +@build_sniff_from_prefix +class PSMS(Tabular): + """ + Percolator tab-delimited output (PSM level, .psms) as used in OpenMS + https://github.com/OpenMS/OpenMS/blob/0fc8765670a0ad625c883f328de60f738f7325a4/src/openms/source/FORMAT/FileHandler.cpp#L453 + see also http://www.kojak-ms.org/docs/percresults.html + + Note that the data rows can have more columns than the header line + since ProteinIds are listed tab-separated. + """ + file_ext = "psms" + + def __init__(self, **kwd): + super(PSMS, self).__init__(**kwd) + self.column_names = ["PSMId", "score", "q-value", "posterior_error_prob", "peptide", "proteinIds"] + + def display_peek(self, dataset): + """Returns formated html of peek""" + return self.make_html_table(dataset, column_names=self.column_names) + + def sniff_prefix(self, file_prefix): + fh = file_prefix.string_io() + line = [_.strip() for _ in fh.readline().split("\t")] + if len(line) == 6 and line == self.column_names: + return True + return False + + class PepXmlReport(Tabular): """pepxml converted to tabular report""" edam_data = "data_2536" From 2dc2eda41f600b148cd4ad44ccb4486d0e4eaf6b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 14 Jan 2020 16:32:28 +0100 Subject: [PATCH 110/324] mrm: as subclass of Tabular --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 5cab3114fc3..d5d433ff601 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -228,6 +228,8 @@ + + From 5e084cb351a652d7f87f7b6c995dc0bd2042ef12 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 27 Jan 2020 14:19:35 +0100 Subject: [PATCH 111/324] suggested changes from review - no sniffing for dta - remove superficial len checks - schema links to OpenMS with commit --- .../config/sample/datatypes_conf.xml.sample | 2 -- lib/galaxy/datatypes/proteomics.py | 26 ++++--------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index d5d433ff601..76867b2a7db 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -230,7 +230,6 @@ - @@ -856,7 +855,6 @@ - diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 536bfc202a4..7484d416b60 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -131,7 +131,7 @@ class Kroenik(Tabular): def sniff_prefix(self, file_prefix): fh = file_prefix.string_io() line = [_.strip() for _ in fh.readline().split("\t")] - if len(line) != len(self.column_names) or line != self.column_names: + if line != self.column_names: return False line = fh.readline().split("\t") try: @@ -161,7 +161,7 @@ class PepList(Tabular): def sniff_prefix(self, file_prefix): fh = file_prefix.string_io() line = [_.strip() for _ in fh.readline().split("\t")] - if len(line) == 5 and line == self.column_names: + if line == self.column_names: return True return False @@ -189,7 +189,7 @@ class PSMS(Tabular): def sniff_prefix(self, file_prefix): fh = file_prefix.string_io() line = [_.strip() for _ in fh.readline().split("\t")] - if len(line) == 6 and line == self.column_names: + if line == self.column_names: return True return False @@ -233,7 +233,6 @@ class ProtXmlReport(Tabular): return self.make_html_table(dataset, column_names=self.column_names) -@build_sniff_from_prefix class Dta(TabularData): """dta The first line contains the singly protonated peptide mass (MH+) and the @@ -264,21 +263,6 @@ class Dta(TabularData): dataset.metadata.column_names = ['m/z', 'intensity'] dataset.metadata.delimiter = " " - def sniff_prefix(self, file_prefix): - has_data = False - for line in file_prefix.line_iterator(): - line = line.strip().split(" ") - if len(line) != 2: - return False - try: - line = [float(_) for _ in line] - except ValueError: - return False - if not all(_ >= 0 for _ in line): - return False - has_data = True - return has_data - @build_sniff_from_prefix class Dta2d(TabularData): @@ -665,8 +649,8 @@ class XquestSpecXML(ProteomicsXml): class QCML(ProteomicsXml): """qcml - https://github.com/OpenMS/OpenMS/blob/develop/share/OpenMS/SCHEMAS/mzQCML_0_0_5.xsd - https://github.com/OpenMS/OpenMS/blob/develop/share/OpenMS/SCHEMAS/qcML_0.0.7.xsd + https://github.com/OpenMS/OpenMS/blob/113c49d01677f7f03343ce7cd542d83c99b351ee/share/OpenMS/SCHEMAS/mzQCML_0_0_5.xsd + https://github.com/OpenMS/OpenMS/blob/3cfc57ad1788e7ab2bd6dd9862818b2855234c3f/share/OpenMS/SCHEMAS/qcML_0.0.7.xsd """ file_ext = "qcml" blurb = 'QualityAssessments to runs' From 86eb5dfb1a8f9f93722de444c9cd43969f49cda5 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 14 Feb 2020 15:54:36 +0100 Subject: [PATCH 112/324] sniffer tests for openms related data types --- lib/galaxy/datatypes/binary.py | 36 +++++++- lib/galaxy/datatypes/proteomics.py | 72 ++++++++++++++- lib/galaxy/datatypes/test/test.dta2d | 9 ++ lib/galaxy/datatypes/test/test.edta | 7 ++ lib/galaxy/datatypes/test/test.kroenik | 23 +++++ lib/galaxy/datatypes/test/test.mztab | 28 ++++++ lib/galaxy/datatypes/test/test.mztab2 | 92 +++++++++++++++++++ lib/galaxy/datatypes/test/test.osw | Bin 0 -> 102400 bytes lib/galaxy/datatypes/test/test.peplist | 36 ++++++++ lib/galaxy/datatypes/test/test.pqp | Bin 0 -> 65536 bytes lib/galaxy/datatypes/test/test.psms | 122 +++++++++++++++++++++++++ lib/galaxy/datatypes/test/test.sqmass | Bin 0 -> 90112 bytes 12 files changed, 417 insertions(+), 8 deletions(-) create mode 100644 lib/galaxy/datatypes/test/test.dta2d create mode 100644 lib/galaxy/datatypes/test/test.edta create mode 100644 lib/galaxy/datatypes/test/test.kroenik create mode 100644 lib/galaxy/datatypes/test/test.mztab create mode 100644 lib/galaxy/datatypes/test/test.mztab2 create mode 100644 lib/galaxy/datatypes/test/test.osw create mode 100644 lib/galaxy/datatypes/test/test.peplist create mode 100644 lib/galaxy/datatypes/test/test.pqp create mode 100644 lib/galaxy/datatypes/test/test.psms create mode 100644 lib/galaxy/datatypes/test/test.sqmass diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 9a48239111a..16735eb7731 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1571,7 +1571,17 @@ class MzSQlite(SQlite): class PQP(SQlite): - """Class describing a Peptide query parameters file""" + """ + Class describing a Peptide query parameters file + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.pqp') + >>> PQP().sniff(fname) + True + >>> fname = get_test_fname('test.osw') + >>> PQP().sniff(fname) + False + """ file_ext = "pqp" def set_meta(self, dataset, overwrite=True, **kwd): @@ -1590,7 +1600,17 @@ class PQP(SQlite): class OSW(PQP): - """Class describing OpenSwath output""" + """ + Class describing OpenSwath output + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.osw') + >>> OSW().sniff(fname) + True + >>> fname = get_test_fname('test.sqmass') + >>> OSW().sniff(fname) + False + """ file_ext = "osw" def set_meta(self, dataset, overwrite=True, **kwd): @@ -1604,7 +1624,17 @@ class OSW(PQP): class SQmass(SQlite): - """Class describing a Sqmass database """ + """ + Class describing a Sqmass database + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.sqmass') + >>> SQmass().sniff(fname) + True + >>> fname = get_test_fname('test.pqp') + >>> SQmass().sniff(fname) + False + """ file_ext = "sqmass" def set_meta(self, dataset, overwrite=True, **kwd): diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 7484d416b60..2e6a720e9ae 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -55,12 +55,22 @@ class Wiff(Binary): @build_sniff_from_prefix class MzTab(Text): - """exchange format for proteomics and metabolomics results""" + """ + exchange format for proteomics and metabolomics results + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.mztab') + >>> MzTab().sniff(fname) + True + >>> fname = get_test_fname('test.mztab2') + >>> MzTab().sniff(fname) + False + """ edam_data = "data_3681" file_ext = "mztab" # section names (except MTD) __sections = ["PRH", "PRT", "PEH", "PEP", "PSH", "PSM", "SMH", "SML", "COM"] - __version_re = r"([0-9]{1,2})(\.[0-9])?(\.[0-9])?$" + __version_re = r"(1)(\.[0-9])?(\.[0-9])?" def __init__(self, **kwd): super(MzTab, self).__init__(**kwd) @@ -97,10 +107,20 @@ class MzTab(Text): class MzTab2(MzTab): - """exchange format for proteomics and metabolomics results""" + """ + exchange format for proteomics and metabolomics results + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.mztab2') + >>> mztab2().sniff(fname) + True + >>> fname = get_test_fname('test.mztab') + >>> mztab2().sniff(fname) + False + """ file_ext = "mztab2" __sections = ["SMH", "SML", "SFH", "SMF", "SEH", "SME", "COM"] - __version_re = r"([0-9]{1,2})(\.[0-9])?(\.[0-9])?-M$" + __version_re = r"(2)(\.[0-9])?(\.[0-9])?-M$" def __init__(self, **kwd): super(MzTab2, self).__init__(**kwd) @@ -117,7 +137,17 @@ class MzTab2(MzTab): @build_sniff_from_prefix class Kroenik(Tabular): - """ Kroenik (HardKloer sibling) files""" + """ + Kroenik (HardKloer sibling) files + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.kroenik') + >>> Kroenik().sniff(fname) + True + >>> fname = get_test_fname('test.peplist') + >>> Kroenik().sniff(fname) + False + """ file_ext = "kroenik" def __init__(self, **kwd): @@ -147,6 +177,14 @@ class PepList(Tabular): """ Peplist file as used in OpenMS https://github.com/OpenMS/OpenMS/blob/0fc8765670a0ad625c883f328de60f738f7325a4/src/openms/source/FORMAT/FileHandler.cpp#L432 + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.peplist') + >>> PepList().sniff(fname) + True + >>> fname = get_test_fname('test.psms') + >>> PepList().sniff(fname) + False """ file_ext = "peplist" @@ -175,6 +213,14 @@ class PSMS(Tabular): Note that the data rows can have more columns than the header line since ProteinIds are listed tab-separated. + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.psms') + >>> PSMS().sniff(fname) + True + >>> fname = get_test_fname('test.kroenik') + >>> PSMS().sniff(fname) + False """ file_ext = "psms" @@ -277,6 +323,14 @@ class Dta2d(TabularData): Note: sniffer detects (tab or space separated) dta2d files with correct header, wo header seems to generic + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.dta2d') + >>> Dta2d().sniff(fname) + True + >>> fname = get_test_fname('test.edta') + >>> Dta2d().sniff(fname) + False """ file_ext = "dta2d" comment_lines = 0 @@ -370,6 +424,14 @@ class Edta(TabularData): (or rt1). All other column names for sub-features are faithfully ignored. Note the sniffer only detects files with header. + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.edta') + >>> Edta().sniff(fname) + True + >>> fname = get_test_fname('test.dta2d') + >>> Edta().sniff(fname) + False """ file_ext = "edta" comment_lines = 0 diff --git a/lib/galaxy/datatypes/test/test.dta2d b/lib/galaxy/datatypes/test/test.dta2d new file mode 100644 index 00000000000..42840611a97 --- /dev/null +++ b/lib/galaxy/datatypes/test/test.dta2d @@ -0,0 +1,9 @@ +#MZ SEC INT +500.0 0 50 +600.0 1 100 +700.0 2 200 +800.0 3 400 +900.0 4 200 +1000.0 5 100 +1100.0 6 50 +900.0 2 200 diff --git a/lib/galaxy/datatypes/test/test.edta b/lib/galaxy/datatypes/test/test.edta new file mode 100644 index 00000000000..66ae17a2b28 --- /dev/null +++ b/lib/galaxy/datatypes/test/test.edta @@ -0,0 +1,7 @@ +RT mz Int charge Meta2 +10 114 2342 1 +10 115 232 2 +10 116 523 2 +14 220 343 1 value +14 431.1 343 2 +15 543.2393 343 3 b diff --git a/lib/galaxy/datatypes/test/test.kroenik b/lib/galaxy/datatypes/test/test.kroenik new file mode 100644 index 00000000000..b554c1f622d --- /dev/null +++ b/lib/galaxy/datatypes/test/test.kroenik @@ -0,0 +1,23 @@ +File First Scan Last Scan Num of Scans Charge Monoisotopic Mass Base Isotope Peak Best Intensity Summed Intensity First RTime Last RTime Best RTime Best Correlation Modifications +20060502data08_exc_RTf.mzXML 1480 1578 15 3 1345.607960 449.543300 60834.925781 506348.656250 521.106018 553.833008 533.018982 0.991700 _ +20060502data08_exc_RTf.mzXML 1487 1557 11 2 1345.608400 673.811500 26904.589844 199969.046875 523.559998 546.731018 533.018982 0.995300 _ +20060502data08_exc_RTf.mzXML 1557 1620 10 3 2002.810800 668.945100 38904.792969 217929.312500 546.731018 567.784973 553.833008 0.986300 _ +20060502data08_exc_RTf.mzXML 1571 1592 4 2 2002.811525 1002.913800 4603.663086 14353.689453 551.427979 558.577026 556.184998 0.978400 _ +20060502data08_exc_RTf.mzXML 1592 1690 15 3 1772.892100 591.971200 91715.507813 502103.500000 558.577026 590.963013 567.784973 0.986900 _ +20060502data08_exc_RTf.mzXML 1599 1683 13 4 1772.891285 444.230100 72974.398438 417335.125000 560.914001 588.521973 570.002014 0.974700 _ +20060502data08_exc_RTf.mzXML 1599 1676 11 2 1772.889336 887.452600 22342.041016 122096.265625 560.914001 586.155029 567.784973 0.994600 _ +20060502data08_exc_RTf.mzXML 1690 1774 13 2 1401.655615 701.834900 37010.062500 262936.343750 590.963013 619.705017 605.291992 0.995500 _ +20060502data08_exc_RTf.mzXML 1746 1816 11 2 1065.545100 533.779800 48403.296875 347601.562500 610.046997 634.202026 622.135986 0.979800 _ +20060502data08_exc_RTf.mzXML 1816 1851 6 4 2095.873767 525.226300 17985.734375 73192.179688 634.202026 646.385986 641.471008 0.986700 _ +20060502data08_exc_RTf.mzXML 1858 1879 4 4 1904.987400 477.504500 7037.772461 16893.957031 648.786011 656.114014 656.114014 0.962100 _ +20060502data08_exc_RTf.mzXML 1879 1907 4 2 994.581075 498.297700 28557.644531 92082.468750 656.114014 665.525024 660.861023 0.996100 _ +20060502data08_exc_RTf.mzXML 1886 1994 15 3 1506.688193 503.236500 223091.640625 1329185.625000 658.479980 693.369995 670.021973 0.989100 _ +20060502data08_exc_RTf.mzXML 1893 1955 10 2 1506.689740 754.352500 49197.664063 307397.281250 660.861023 680.966003 672.174988 0.990500 _ +20060502data08_exc_RTf.mzXML 1907 2056 23 4 2239.081209 561.027900 189284.390625 1359320.875000 665.525024 713.080994 685.143005 0.973600 _ +20060502data08_exc_RTf.mzXML 1914 1994 13 2 1336.644692 669.329400 174888.703125 994234.375000 667.744019 693.369995 676.612000 0.990700 _ +20060502data08_exc_RTf.mzXML 1921 1988 10 2 1478.675160 740.344800 119720.226563 577115.250000 670.021973 691.309998 678.859985 0.985600 _ +20060502data08_exc_RTf.mzXML 1928 1948 3 2 1697.803500 849.907800 3211.451660 7545.761719 672.174988 678.859985 672.174988 0.927400 _ +20060502data08_exc_RTf.mzXML 1935 1955 4 2 1127.618850 564.816500 67462.703125 183348.593750 674.471985 680.966003 680.966003 0.950400 _ +20060502data08_exc_RTf.mzXML 1941 2021 12 3 2239.081067 747.701800 67137.210938 463090.031250 676.612000 701.872009 683.143982 0.991200 _ +20060502data08_exc_RTf.mzXML 1948 2028 13 2 1583.758454 792.886000 36854.781250 262471.843750 678.859985 704.109009 691.309998 0.993400 _ +20060502data08_exc_RTf.mzXML 1955 2021 11 3 1583.757945 528.926500 128648.492188 829892.062500 680.966003 701.872009 691.309998 0.990400 _ diff --git a/lib/galaxy/datatypes/test/test.mztab b/lib/galaxy/datatypes/test/test.mztab new file mode 100644 index 00000000000..6c74f6d27ae --- /dev/null +++ b/lib/galaxy/datatypes/test/test.mztab @@ -0,0 +1,28 @@ +MTD mzTab-version 1.0 rc5 +MTD mzTab-mode Summary +MTD mzTab-type Identification +MTD mzTab-ID Cytidine +MTD description LC-MS/MS Reference Standard +MTD sample_processing[1] [MS, MS:1000544, Conversion to mzML, ]|[MS, MS:1000035, Peak picking, ]|[MS, MS:1001994, Top Hat baseline reduction, ]|[MS, MS:1000782, Savitzky-Golay smoothing, ]|[MS, MS:1000594, Low intensity data point removal, ] +MTD instrument[1]-name [MS, MS:1000483, Thermo Fisher Scientific instrument model, LTQ Orbitrap Velos] +MTD instrument[1]-source [MS, MS:1000008, Ionization Type, ESI] +MTD instrument[1]-analyzer[1] [MS, MS:1000443, Mass Analyzer Type, Orbitrap] +MTD instrument[1]-detector [MS, MS:1000453, Detector, Dynode Detector] +MTD software[1] [MS, MS:1002205, ProteoWizard msconvert, ] +MTD software[1]-setting[1] Peak Picking MS1 +MTD software[1] [MS, MS:1001457, data processing software, MassCascade-KNIME] +MTD smallmolecule_search_engine_score[1] [MS, MS:1001153, search engine specific score,] +MTD contact[1]-name Stephan Beisken +MTD contact[1]-affiliation European Bioinformatics Institute (EMBL-EBI) +MTD contact[1]-email beiken@ebi.ac.uk +MTD uri[1] http://www.ebi.ac.uk/metabolights/MTBLS38 +MTD fixed_mod[1] [MS, MS:1002453, No fixed modifications searched, ] +MTD variable_mod[1] [ , , CHEMMOD:2M+H, ] +MTD variable_mod[2] [ , , CHEMMOD:M-C5H8O4, ] +MTD ms_run[1]-format [MS, MS:1000584, Proteomics Standards Inititative mzML file format, mzML file] +MTD ms_run[1]-location ftp://ftp.ebi.ac.uk/pub/databases/metabolights/studies/public/MTBLS38/cytidine.mzML +MTD ms_run[1]-id_format [MS, MS:1000767, Native spectrum identifier format, ] +MTD ms_run[1]-fragmentation_method [MS, MS:1000133, Collision-induced dissociation, ] + +SMH identifier chemical_formula smiles inchi_key description exp_mass_to_charge calc_mass_to_charge charge retention_time taxid species database database_version spectra_ref search_engine best_search_engine_score[1] modifications +SML CHEBI:17562 C9H13N3O5 Nc1ccn([C@@H]2O[C@H](CO)[C@@H](O)[C@H]2O)c(=O)n1 UHDGCWIWMRVCDJ-XVFCMESISA-N Cytidine 244.0928 null 1 193.25 null null ChEBI 109 null [MS, MS:1001083, ms-ms search, MassBank] 977 CHEMMOD:2M+H,CHEMMOD:M-C5H8O4 diff --git a/lib/galaxy/datatypes/test/test.mztab2 b/lib/galaxy/datatypes/test/test.mztab2 new file mode 100644 index 00000000000..d38a03df942 --- /dev/null +++ b/lib/galaxy/datatypes/test/test.mztab2 @@ -0,0 +1,92 @@ +COM Meta data section +MTD mzTab-version 2.0.0-M +MTD mzTab-ID mzTab-GCxGC-MS +MTD description Minimal sample file for GCxGC-MS quantification of small molecules between two experiments +MTD instrument[1]-name [MS, MS:1001945, Pegasus 4D, ] +MTD instrument[1]-source [MS, MS:1000389, electron Ionization, ] +MTD instrument[1]-analyzer[1] [MS, MS:1000084, time-of-flight, ] +MTD instrument[1]-detector [MS, MS:1000114, microchannel plate detector, ] +MTD software[1] [MS, MS:1001799, ChromaTOF software, 3.21] +MTD software[1]-setting[1] baseline=0.2 +MTD software[1]-setting[2] dbMatchTreshold=900 +MTD sample[1]-species[1] [NCBITaxon, NCBITaxon:9606, Homo sapiens, ] +MTD sample[1]-cell_type[1] [CL, CL:0000233, platelet, ] +MTD sample[1]-description Unstimulated human blood platelets +MTD sample[2]-species[1] [NCBITaxon, NCBITaxon:9606, Homo sapiens, ] +MTD sample[2]-cell_type[1] [CL, CL:0000233, platelet, ] +MTD sample[2]-description Unstimulated human blood platelets +MTD ms_run[1]-location file://c:/data/control.mzML +MTD ms_run[1]-format [MS, MS:1000584, mzML file, ] +MTD ms_run[1]-id_format [MS, MS:1000776, scan number only nativeID format, ] +MTD ms_run[1]-scan_polarity[1] [MS, MS:1000130, positive scan, ] +MTD ms_run[2]-location file://c:/data/treatment.mzML +MTD ms_run[2]-format [MS, MS:1000584, mzML file, ] +MTD ms_run[2]-id_format [MS, MS:1000776, scan number only nativeID format, ] +MTD ms_run[2]-scan_polarity[1] [MS, MS:1000130, positive scan, ] +MTD assay[1]-sample_ref sample[1] +MTD assay[1]-ms_run_ref ms_run[1] +MTD assay[2]-sample_ref sample[2] +MTD assay[2]-ms_run_ref ms_run[2] +MTD study_variable[1] Untreated +MTD study_variable[1]-assay_refs assay[1] +MTD study_variable[1]-description drug response control +MTD study_variable[1]-average_function [MS, MS:1002962, mean, ] +MTD study_variable[1]-variation_function [MS, MS:1002885, standard error, ] +MTD study_variable[2] Treated +MTD study_variable[2]-assay_refs assay[2] +MTD study_variable[2]-description drug response treatment +MTD study_variable[2]-average_function [MS, MS:1002962, mean, ] +MTD study_variable[2]-variation_function [MS, MS:1002885, standard error, ] +MTD cv[1]-label MS +MTD cv[1]-full_name PSI-MS controlled vocabulary +MTD cv[1]-version 20-06-2018 +MTD cv[1]-uri https://www.ebi.ac.uk/ols/ontologies/ms +MTD cv[2]-label NCBITaxon +MTD cv[2]-full_name An ontology representation of the NCBI organismal taxonomy Ontology +MTD cv[2]-version 2018-03-02 +MTD cv[2]-uri https://www.ebi.ac.uk/ols/ontologies/ncbitaxon +MTD cv[3]-label CL +MTD cv[3]-full_name The Cell Ontology is a structured controlled vocabulary for cell types in animals. +MTD cv[3]-version 2017-12-11 +MTD cv[3]-uri https://www.ebi.ac.uk/ols/ontologies/cl +MTD cv[4]-label PRIDE +MTD cv[4]-full_name PRIDE PRoteomics IDEntifications (PRIDE) database controlled vocabulary +MTD cv[4]-version 14-06-2018 +MTD cv[4]-uri https://www.ebi.ac.uk/ols/ontologies/pride +MTD cv[5]-label CHEBI +MTD cv[5]-full_name Chemical Entities of Biological Interest +MTD cv[5]-version 08-02-2019 +MTD cv[5]-uri https://www.ebi.ac.uk/ols/ontologies/chebi +MTD database[1] [, ,Golm Metabolite Database, ] +MTD database[1]-prefix GMD +MTD database[1]-version 2.3 +MTD database[1]-uri http://gmd.mpimp-golm.mpg.de/ +MTD database[2] [, , no database, null] +MTD database[2]-prefix null +MTD database[2]-uri null +MTD database[2]-version Unknown +MTD derivatization_agent[1] [,,Methoxylamine hydrochloride,] +MTD derivatization_agent[2] [CHEBI, CHEBI:85064, N-methyl-N-(trimethylsilyl)trifluoroacetamide,] +MTD small_molecule-identification_reliability [MS, MS:1002896, compound identification confidence level, ] +MTD id_confidence_measure[1] [MS, MS:1002890, fragmentation score, ] +MTD small_molecule-quantification_unit [PRIDE, PRIDE:0000330, Arbitrary quantification unit, ] +MTD small_molecule_feature-quantification_unit [PRIDE, PRIDE:0000330, Arbitrary quantification unit, ] +MTD quantification_method [,,baseline-corrected intensity quantification,] +MTD custom[1] [MS, MS:1000901, retention time normalization standard, n-alkanes C10–C36] + +COM Small molecule summary rows (similar to Protein section). +COM Evidences (e.g. multiple modifications, adducts incl. charge variants are summarized). +COM For most use cases this summary lines might be sufficient. +SMH SML_ID SMF_ID_REFS database_identifier chemical_formula smiles inchi chemical_name uri theoretical_neutral_mass adduct_ions reliability best_id_confidence_measure best_id_confidence_value abundance_assay[1] abundance_study_variable[1] abundance_variation_study_variable[1] abundance_assay[2] abundance_study_variable[2] abundance_variation_study_variable[2] +SML 1 1 | 2 GMD:cd7993ea-ad14-452a-a907-33376cc98790 C18H36O2 CCCCCCCCCCCCCCCCCC(O)=O InChI=1S/C18H36O2/c1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18(19)20/h2-17H2,1H3,(H,19,20) Octadecanoic acid http://identifiers.org/gmd/cd7993ea-ad14-452a-a907-33376cc98790 284.478 [M+H]1+ 2 [MS, MS:1002890, fragmentation score, ] 978 805.16 805.16 0 589.9 589.9 0 + +COM Small molecule feature rows (only reported in Complete Quantification files and if feature information like e.g. mass traces are important) +SFH SMF_ID SME_ID_REFS SME_ID_REF_ambiguity_code adduct_ion isotopomer exp_mass_to_charge charge retention_time_in_seconds retention_time_in_seconds_start retention_time_in_seconds_end abundance_assay[1] abundance_assay[2] opt_global_retention_time_nd opt_global_retention_time_nd_window_start opt_global_retention_time_nd_window_end +SMF 1 1 null [M+H]1+ null 285.484 1 1564.47 1559.45 1564.48 805.16 805.16 1562 | 2.47 1557 | 2.45 1562 | 2.48 +SMF 2 2 null [M+H]1+ null 285.484 1 1564.48 1554.45 1569.47 589.9 589.9 1562 | 2.48 1552 | 2.45 1567| 2.47 + +COM Small molecule evidence rows for parent ions. Analog to PSM. +COM Primary use case: report single hits from spectral library or accurate mass searches without quantification. +SEH SME_ID evidence_input_id database_identifier chemical_formula smiles inchi chemical_name uri derivatized_form adduct_ion exp_mass_to_charge charge theoretical_mass_to_charge spectra_ref identification_method ms_level id_confidence_measure[1] rank +SME 1 ms_run[1]:scan=8 GMD:f634c736-39e8-4323-8155-fa3cc26ac9e3 C18H36O2 CCCCCCCCCCCCCCCCCC(O)=O InChI=1S/C18H36O2/c1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18(19)20/h2-17H2,1H3,(H,19,20) Octadecanoic acid (1TMS) http://identifiers.org/gmd.analyte/f634c736-39e8-4323-8155-fa3cc26ac9e3 [CHEBI, CHEBI:51088, trimethylsilyl group, 1] [M+H]+ 356.6588 1 356.659 ms_run[1]:scan=8 [,, ChromaTOF database search,] [MS, MS:1000511, ms level, 1] 957 1 +SME 2 ms_run[2]:scan=23 GMD:f634c736-39e8-4323-8155-fa3cc26ac9e3 C18H36O2 CCCCCCCCCCCCCCCCCC(O)=O InChI=1S/C18H36O2/c1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18(19)20/h2-17H2,1H3,(H,19,20) Octadecanoic acid (1TMS) http://identifiers.org/gmd.analyte/f634c736-39e8-4323-8155-fa3cc26ac9e3 [CHEBI, CHEBI:51088, trimethylsilyl group, 1] [M+H]+ 356.6589 1 356.659 ms_run[2]:scan=23 [,, ChromaTOF database search,] [MS, MS:1000511, ms level, 1] 972 1 diff --git a/lib/galaxy/datatypes/test/test.osw b/lib/galaxy/datatypes/test/test.osw new file mode 100644 index 0000000000000000000000000000000000000000..355afa48456414197b6eb85a87a95973f72714a5 GIT binary patch literal 102400 zcmeI53w#vSy~k&EcOJXh-IIixg@i~bLK6+j5`iKYW|m|UR`$j2MnKU8pdVQg^SB;OzwM7Nm>l3h_YZ1g(M9`0bRB7&+*`3+%mzq}HlBrkG(_My{%27^aN=>p}nNfzTa` zen4Y3Wk26;8FTKi&pB0?e67m#h)4Ovkq!bN00JNY0w4eaAOHd&00JNY0wD1H6L6I} zXrV0mf}wv1AOHd&00JNY0w4eaAOHd&00JNY0;dfDJ8Kau#n8ODty5>t@z`j+Sn>@+ z{}4a`1V8`;KmY_l00ck)1V8`;KmY_zLjq>AXjSL`1+tqVUz4NcF!_YMPu``62p|9g zAOHd&00JNY0w4eaAOHd&00O4~0gGT}Mc%^GEob5A)?%^Ht=VFxTa(#jwu)xfEQ;~* zf0Q8~lO1FW*+iC*StNQ2R1hYD00@8p2!H?xfB*=900@8p2!O!6XNCgC3Vzv~+XgxL%LT%v!oBaXeCwD@>fVPaiRH=%FEH z_Y9U3sZn~09;^QU0@*|T|H(1(Iq4(^$Y1Fp0tkQr2!H?xfB*=900@8p2!H?xfWWCh zfX?}|yr6D5Ufo(ab!%3A0az1t1F+Jq;s5_e-2eX&`G~wn-XU+C3KfKnAOHd&00JNY z0w4eaAOHd&00JQJFHL}>uKz60bJX{r;F67Wyl`#I9Wy}kby*` zhX^150w4eaAOHd&00JNY0w4eaAaDv1s1~ejrAuHLA-Nj=teMkiw8{(R>LXiMPcTUz z-=T($)x05Ttk0j2+`EH@ok>Z~kTh28k4s)ZUqh3Wv>1}c8vb#~Yd=azelSf7)pR+& zgq_B!{&C5FAD58aU#Eqin5<$#QeVP9F1X<3gy3E+X8N*LYmG}(tF8;il=-dmCTZd( z=~J~2SS-=SpZ)%yY#wB0NWE@+H}a78q-SvVVw ztQO8nBhk#!HT_w^9RC1-IsO3vQ~Uz}ruYW{Ocwe908w54->R=ovE1)_xWk1 zI#NruyX#U*wWlf;xd+M%Ta&gHrDodPZh7Inq^-8pQmyU*^1>F~)^e+zB?bX;sJw8B zQE0iCnkP^=lZ`Ta2=Tml{dvk}qERHDs>V?=(PWWaYMzCX2`7uRq!wwWU_wTb=2SHk z#nNCD*~4VFCTl>cj5JE@&e}yaD{drZl34vVgXFDX^G#U<*N6sub& zQMV4Ky0tshZINBw+KSYz)uwJmtGX3LbzA8Df2;cbzi3tG|5@@DL;ny!00ck)1V8`; zKmY_l00ck)1V8`;P9*|0teN+Yy63`Ku8l{-!Ir6WB3DmpoiTRqyq`~-HTUY6J2rDp z%T=xC&c4R)Q|JF#@+Cw65I_I~KmY_l00ck)1V8`;KmY_l00d4W0!3!tsy`i2=l^-< z^9*^7Y$TVHTIUgZfB*s@00JNY0w4eaAOHd&00JNY0zW7MgLo^?vw|S7ruS}mZTo=p ze!WN^*z)q^+I6kZ{%oQ2VZFC}+O!!R{Re+GsH^LPAVNLbrE7sNU4^G$Brb>IOkG!~b z^`+9GuC7>DS6319f;#`tlP4MazW+PqL-Hi)fG5BYs*A%I5C8!X009sH0T2KI5C8!X z0D*s10(LXc>Q6g(^+11yK|c-1v-*<>i+Z3xZ!oI|`cnjxI{&Xf8_?$eiH#xq$sg%D ze=A9l9w2}K2!H?xfB*=900@8p2!H?xfWT=>pq{t!)RTarGv|Unyp~^TuTz7(cIJb> zWSC-UUDttER=3QSK0EQZePdpKU-~MJqe%LzH&pu0!LDIkq7XU*uMf+!3fa(hz}{vb zB6UXJx@TVR;f8XB#m0-Gz=$F%>ce<>{C(w~`^qO4SD2HT`mkQUgzp@o447V0VM=D| z!+d$dTJKqFy~W)s*kqo>{avt0R%t*1V8`; zKmY_l00cnbI}vD8_`ce3LZoAglt2p+=>PGpVxlx-q_Jzt9W-o|cm1mW^(Pp!eBWB@ zcIpT$?d!Q>^ZfJvARYQW^G%E8nAExancq*p^N6%fd2J%Y6iJ7UA8tDPPj^X&{(kAY z8SCdr>nlDT<=^?VwB)h3?+L$fv9xSl_11$&j!T2+W1-sHYQTKU-@Mz7v7SZez0xf$~wKT~xPrr}td!hNfH7=xW3Y&|UBsvgF; zYxf1d9XodHly}d*>49;KCD|VC;>)LOx?eiO z-8E~p_Gn)4H?=*Un>ehTw#TlDrmAyjd(<>N+x!4+k9Y39VZd*tPww7ub&{RO}2a5Q|s}a`qhVbKld;YjHwLXm-7F25OQvdNP+Aarc%*+ht{B z^h0X9ytwPaYiYauMOx!(mp<%THTc?N{)VE(s{PeZ18@z9Z_06jtLVzjdC(!XfdByoV z$3EvWN0WV;qpWC~?YBkiZH#r2aKc(ETqnHEPY?ne!$>6Y%tNb?5a25lHA4gfAOHd& z00MCWZF5TnS49PT!=TnllV?qBR6Nn{A+}v9aEU=%|cR!rrD)Idf|0Z{vC2q4mcW3$AY6*tRn?KD2)HuOy8NO(@8u z(d$zJ9zVsXjH8$msA~F_@UEfVSZ3y&DO2ZPdBv=mty5!@u4aNqYbAHYkCpw{d({#+YVL z?hK!IWSDld7%f>*Yd$_M!=cAfr6LTl`i`#lS1eeFdw$*1a zj)jy^l-eBgdqN>7BmW=SjaW z)<*pFei_L~%yFfkC-ScP%6u(goZGGMS=+kvf=iOvHc6{s5>duQo;O2CLy%Xj0DS#S zS?qgTv|r)HJ>sUtZv+WxF8#Feyn^7H1O8xxx6!LK#3IU=W+hOc@#9l_viuyp`*Xa@ zQ<0CiUNd)Mhi@x33$9_q*tQ|5Rf^Hyn5g0n7+pPa%gac%n4md6(^PDu^&)HoawNmYM1QQnW`B z>imDHJ<5t9TjOw?&RCFo}&bvF4GDn5}Zm2cC;i|R=B zIdsPbXI{gc`IQ`b4Rd50rk}~4!!UdHIdo5iB6;&&TeZq>eu)n-)HOsWR+L}KCOx53M&X!-vD!)3XpXMy^s;2aX!Ofbjq&pah@)P`p!&~m%B|p(` z_^=M@DWRJu=2sHrb}Ln}4VT-8Uh+2m0-a%=kY9hEp|f4nap~;T@xMiAq3i!Uud;5TI|v{E0w4eaAOHg2JAoyhGQs6`vp24WV2YCS1qPNFthXqI<`hx$ciO?>>Cs@+oy%#Dnqc4y?MWkv!y zPv`eUA`Qw#x%0gFlJlGIjRd2?kV1_NhZUb%%iQPH<$@O}VV@_Id7_%-BhC<9efqF# z<1Phy^ZO$MQ+CU8_8Kaax4RT<@c*MeF*2fry-Fmlih4Ix`%FVHN@uL-RAWkyNzKpQ zr_v-&0zH>(F7T;x`b56kkcsY;D?4F)0SUh+Fv=5+di3s^tCX-m(omp?a4;D4E7Wm^ zdbt=ERjzU(ofZVF86(-fJC7R#0yKk&14s(uE}~czSxOiRn4<-jGvC&7xEPjd?SpVI`oqYkH~S$e5&7&EoMt(r#c& z7HG5O$-9<2W%rCcH!$}39V9C!QF*2~j5^_z9Rn|f*yy6nmBFWl&IJb2f;6L;0UWxXrasA#d(1cE{X z!AL0}mJLyPgO{QTRGxT1@B#=bA@!kx2MQvQkir86sgwd$!$q`65G1I?cjmUUYwt#= zm5|;4C_ZQAoNvyY^PBP78_%5GJX^o8d#-T)(&dkB?-oueIYkJiR46EllIE`*e??1* zSM+FubCFn|wwhMnar(6^{YxS1b%h?FpSe$GUveMMZe-497t+tAzL|a^r8wKx>(1)5 zCQuFm2tWV=5P-m&DsXkxwerW0i)$x#w|6d{>vqb`&04qCYL3>Z((%e&uh1>8*Ex+N zmTT2Qt=TQKJGDl+vsHM|+bT3$-9ochudi}gqu%F;uJ=$_)b6yZy-K&=_&|XhtdC+V z50pC_p5Iom?^fF@$in5~O3+%XQ>`_57kM~8r0R9OO1IYB2#TVk)~a5!TRXG0r{Zj{ zynjW#w%*~+_qk2f2p;pvG_Np!G@ z@|EE|87uF3V))>m?sdFMud~_e?AO7`y6n_0PDak`WE{Mwn;%VC`D4e#7lJFb9R9G#P#sBMNy3uL%+WmTY-K&pXyfMX{F5g*~R8-gL&c4oyX*2H~n|7VWW9t{=)y(j2 zCwY~{75z%9(Qfsc)#xNnuSxAMxTl5XQ47O&2ocZm;Be6=!Iaj*XW;VbsD{Td9tQat9zRhh5LM`ztcAJm8EHCAU<|6 zT62bXnO1&nO$^uk0gi~G@4N7ud}@U?MldyO27Yv;J@&_SE5EoXt{w@x7QE_&(Z={; zgSwgUy=J3Tt(~cP)&8b;w&yh~ll*t8-^cr;Vdl$=)4ZdR6~h&CnA5EM>C<9(e{x8D z{u$HtYR&jk4_{v=iuL`5Jh|_3bj%6O)u^$F@|A(`k%ztT{QEr#gVpgP2ECk#937=x zsd$^4`~&&Um+VP(CSgd!FByK%H2yupmcL76`Ag8>75;|>0uX=z1Rwwb2tWV=5P$## zAOL|wBajigy=1p9U)nu);nA}EqrRa3DEto#1Rwwb2tWV=5P$##AOHafKmY=VP(agc zNB;faqF*WWZ~7;_N`In1&+<|VQEHeuM@si-8%<^BH_J3m{a!4Zz_vzW@JbaR2|`=oR`M{hDqbgaM+D z5P$##AOHafKmY;|fB*y_0D-r(fWf!^3)3|C-hW~6!kMlceD}Z5G=p#d7plDf-6e*Q4>#p}G>7)2U!WISVk5c#<)KFugnk0awb{Da8Pz7j^heBNJ{ zc{;wZpYg2zLF8v13L}4i%3t2DI(8TtP2nE|KK9cv@I^mY?iG%+lDFm1qrkp*W%u&7 zAFi(bMULwFjd=e5+x!UtnEyYpLka;1KmY;|fB*y_009U<00Iy=C<1cYzghGD+j9Q@ zVE(^(`eHEu|EqM1UZUU7FX;yTgr21z()Z}w^bLB7zCvH(AS@7o00bZa0SG_<0uX=z z1Rwwb2)q#iW1iox&zZ&?XF1czaYm*=oVkYKa+WoWEN2ggsY$Uukg7ACvl+rY+7TD|5yBOM|Qa zW9#aUY8=f4m5#qB72jJLd_H)zekZj^$sOI|d&XLIla<-xyT@A1CR@!EPbFK;BzH^~ z?<);fVe*Hp4qr}(bY;NwwWk2)Dav4H*l(%_M}(W7><&SIRMxXmM0 zP#28fWSbAgO`6Hw43_DRnlzGiI?HrMP3p-eHHK-&O=`*ARA#BhO&(D*&S(UzvL3g1 zm&n+%D(EDegtH&)q~!De^Yo;`PygQ@J^xQP`3ZpM=tuMnJ!4Z=cD6mjNo5CqZb`o4-Kh z9l)ree`s~QXa`2?(hg$LG0qZR2AA;VK;k`N*=QGq)_By{Q9 zBWjO2MAQ%UxiBTd>uS29a;!8a zJUKjuNV2;!_Gw6^T0O#g`+0E#xLziHYE<@DdT5eTnlj~(>Ys*`C@S=bCgh5^d=bxw zFBb8W@ZH28T&w z>+7Rgm@H&!Ot`O*>*g=?^AUd7Q073aMfeEnDsvHf5C^n~ z!}s^{GV|s7ig@l^u^(69&*i&uIU;-{b?$N(^2Uf1SRDK;;MG~^;wIt+xF{ICxuUT? zim0FL;_|sdk3glT@%((eI-eyXUSIUPyS#kHbA`XV$KTgid5?+QhCyQw7=UJs?jrJW z_jeQd3Kb8`k9ZV?|0kS3IqO+88r#qil{+btR_-0;ZF-ONmG>BT_95QmJZ~3aAZHvm zki+-s>aEVRgrbl6xO4!Q=<&|Z?5biif=*?RGenWj<0vr(pLmHwl4S9UpT)|rB(_ri ziQz}35(6?tk@ttQ$cGJDkBmPQcbyg#DG>(^$JNWn^+UoY2^1PzQxlbYbVtRXsPb{7 zf7|3SJTETa#k+gVc1v(R*M374D%)Q3bL}Vcw!&9bxebZN*3&~X>=l`HpSZdo$vtxH zF8#NO5}~_}zpiuIZwFw{(~{EY_j7aMi+%CKM#vKd{#O5Ujk6$A*=~BDYg~7lVyo^g zVkS)|ezlp2(%9PCs61HFTB4(fZ&Hs|etgbT^r`X?>298zk2iik^7F$_vmg3hEDH2h zoN>&4XB4R688G`?K(Izkj!jUnQD3dDrS?!QUo8qRgargZ00ck)1V8`;KmY_l00jQ> z0&*6UhF{2JWb~&{QwyZ~XFfs5tcfdGjYc+t5@*W|8wOUrn(;7r{03Hwam$vC4SQd| z9w+h9a-7tDtRwl-!;KG)409dmI#5l`b>N+$cZQbz)HFWY$Eo6}N5Dk=x_uQRPS1Y* zw&T}`bK9c#1?k-kwB2=ekbao!oR;9mYZ@C^I|5(G@6PZr8UC~_s_9fZ4JjWy+Sr&u z#WO-ZDJmp9Bi|5&d`n2pw&c8TGVdVw(|+IL?7GDoDOsXx=ucPVE;k?(x!XO9+SKkXZrM{1a8;D|^D5uFTW(dHG35R5W#mhpp(B!5X~zr2T2uQY7dX~Q z#wwlplKwc~z7!h5zz%GL)I*dZ=RR4rbB{GnLWEi+7MwEaMV#Y zE;D#?{)H0rN9`T5r>_qct?H0$PsaA=DOjJy`TS#bRxI}C%j*3f2l9?;soW?vB-}(O$leqqm zV(ke21q%p(00@8p2!H?xfB*=900@8p2!KE@6d;r748`?-1`ENzU;zOT009sH0T2Lz zf26>oSd>CvGdZ>3ad2?_rLkJG;$|N^v3OY(JAu@9!$a=YL5AZhqP~BdO`U;k+gH1` zpWBJDCL5cVQ>4N5*~2HjMp3E#i?xqZvh6Y}7V~m+vYyQ^Zd`sj+06ayoNU3!1N-8h zcdxzNO+|>A>7B!Z4R}Gj}N!RwPf1!5{7tIRz*2aVJ|M--~ZqyREOCp53QeWZtzX= zQhv$kMN^NqWgG~8yJX(O%*)ia}j($%<|Y^dI5rp)Nk$pYqM+ z`Kns=4%&^&EoyIb79Nd$^yHfBIfvYiDn{i#M;lYR_r^IzGkiHUX1DJTUgS=xXH%|* zsN3{oP>L9RW9E)2DYROsk`o#L&lv-HhU*PJU*31g1TOZLoqN6qOwb8}^?-}b!t z=jTm_^@(KOytOi`sVXer;dqR{S>!p!-e%MH^90%*Waq17M)H#y@~t=i&E>o8c_kCV zYfigddbyzG@P*u&b@X2zMAxM8+e^iX&g*9RJh0wT{jHUaAUMb%J;*RC+Gx$Kxp`r# zV-RPZse&ICCFzRl1{~5qzbk#K%UgHh)a~-1`;NI9t=`qQYLT__IT|Y$p7klRq7A&V zPVm}so#0?>VO`5Zoi{eMi3S1l`ybYIOk2O%bcwVo%BeMqdZFsV4GSkv)ma7(Qv`!; z1oGvf`p)XSI~tpgMVv~=$T-HfKF<*Hb{=>;q%l#qtZA?B)`oFm{DftiX;TNrG7p#s z%r4U(eckTlRH3)`_|3)ZeD~c>ZC*+XG`cR;%`E$Na7f0T#DeNCPAon0F78lEh-}lo zw9<8FOV=E6dGnT*zS?WVTLazq)LfNCFS2yy-!*L;#M@|>TeU0eoreQ^Kx*M7%5{U{ zuU{OlmRP*=CU3~Glp&X@E_jsKO&B$4F0yu^;k3P+`&ur(UJon_Zy7IJGR3;q)hIIk zV#|sknmyYVEy$^%NHqpYzb&*KPMr{HQJj*&JgIeWS8Ce+a;v)Ir|rhgsyuT+?78`; zyNIU>MNeyflct(s`y2HYEq)wb!!>LoZ_4-Qt5BkydsHikxPW)T(W5=_evl<(` z>o?MM6Rh*r@}f46&n(a^-t=H^LNar|nSsguOBtoj%$yBl7S^vEBi}>KJrkt1G;*}0 zpCEziJGkCGXU=mCvnsOF^Z9+F1aaD5mXBCe_)b`{R#Lm~&>hR4l3I68oW6H#)0(U6 z{H6&*vSsJbUN|@Zv<<~*JE}{sP0ZnqfOssZ5^AlKK#@YPDDZZR78?@Zo;a2 z7ISzj?)V-U4lG|poNtDy7Ezj1*4l;dF`KaMn!IP~!7du>Y>^Cvq zV*g5W`JMZ+9~W3Yz4G3N+t~Dtd0ly|?CL#)mK0l2O$)cX+H86?c-*P$yP9{KyAO`@ zKe2A$$lUO6OGCb$Vjik}#Bluf;7b0_n_sHmiO%Hm&T3i@WE-w1ApMx)CE6a&*XsEzl%T-;|qb0JIpn{i3> z6#M?}TyoHYgF}l>JQtUF{P5WAJ`CewzT9I{%|biuu4 z3o0tdb}Z)}^(NVmta)=+{i}^x9J9Gk4(;1NY?HnGEjQJ<4J1^fnre5K9pu9NE zS}+m$Ide`-u@{ZrBs+T2(eixRtwYB8F=}gm9JzB_?9bPhZwn1FuZXIgS(|=x=!#c) zqe`!(jVgUHoN?g$lJ#c{TfJ6PNh7yx)H_sc()0ZD?pFMX6&9-%C+UgC$)@4)PrKlAS+H_#1anvt~Z`aCq7$uiJnBdlN zv`xKKzUPO7_w^^A+nb$UmXesTC|+&S=0>UMH@C^Dg?SBEwoLS&{T=)JbuAMEmTOg) zwMpxrqtWY(+%~7qscfIjJm2xks-1RQHO`)x|Epm22=)qlg5Ab0;hV6400@8p2!H?x zfB*=900@8p2!H?x{KW<6DioBiNs&p0(J90hSthZilZh>oOkzzVQ5bYhnZ$;M|CO8A zup;jNSHbS!_y6O60eFTz!Z%?70T2KI5C8!X009sH0T2KI5C8!X_&W-ykWrMPLafOY zVof4b$P9elb^pH#wgtgjuxr>U?0b9@77zdd5C8!X009sH0T2KI5C8!X0D=FJ08532 zpmZ9YPI1l(nno-vN>>TlIkRXLzN}`EyGlsTSyJZcgrY`_L_-+Fvz#-9({Ff`YGEi1 zp?4N^KBx2k|3U=6@_zxV!OF41|4{?sWkCQ0KmY_l00ck)1V8`;KmY_l;2$i&Qc=a{ z{RoQEN%-8qqXTVYQItx^iW0I?2?;N0)g-Cn^M3qUBoc~t(9l-2jYQW(RS`Oh7ovC} zxr3sG*P{^k|1+_Zo%jDg#~x$%up8KA>^xS77s3JpAOHd&00JNY0w4eaAOHd&00JQJ z*Ah_N54UtSd*xVwF+_9KZRJ~_y5y7@Bi Date: Wed, 19 Feb 2020 17:50:00 +0100 Subject: [PATCH 113/324] openms datatypes: fix mzTab --- .../config/sample/datatypes_conf.xml.sample | 2 +- lib/galaxy/datatypes/proteomics.py | 46 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 76867b2a7db..c6bd174ce25 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -228,7 +228,7 @@ - + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 2e6a720e9ae..ae399561485 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -69,8 +69,13 @@ class MzTab(Text): edam_data = "data_3681" file_ext = "mztab" # section names (except MTD) - __sections = ["PRH", "PRT", "PEH", "PEP", "PSH", "PSM", "SMH", "SML", "COM"] - __version_re = r"(1)(\.[0-9])?(\.[0-9])?" + _sections = ["PRH", "PRT", "PEH", "PEP", "PSH", "PSM", "SMH", "SML", "COM"] + # mandatory metadata fields and list of allowed entries (in lower case) + # (or None if everything is allowed) + _man_mtd = {"mzTab-mode": ["complete", "summary"], + "mzTab-type": ['quantification', 'identification'], + "description": None} + _version_re = r"(1)(\.[0-9])?(\.[0-9])?" def __init__(self, **kwd): super(MzTab, self).__init__(**kwd) @@ -87,23 +92,21 @@ class MzTab(Text): def sniff_prefix(self, file_prefix): """ Determines whether the file is the correct type. """ has_version = False - has_mode = False - has_type = False - + found_man_mtd = set() + cnt = 0 contents = file_prefix.string_io() - while True: - line = contents.readline().split("\t") + for line in contents: + if re.match(r"^\s*$", line): + continue + line = line.strip("\r\n").split("\t") if line[0] == "MTD": - if line[1] == "mzTab-version" and re.match(self.__version_re, line) is not None: + if line[1] == "mzTab-version" and re.match(self._version_re, line[2]) is not None: has_version = True - elif line[1] == "mzTab-mode" and line[2].lowercase() in ["complete", "summary"]: - has_mode = True - elif line[1] == "mzTab-type" and line[2].lowercase() in ['quantification', 'identification']: - has_type = True - elif not line[0] in self.__sections: + elif line[1] in self._man_mtd and (self._man_mtd[line[1]] is None or line[2].lower() in self._man_mtd[line[1]]): + found_man_mtd.add(line[1]) + elif not line[0] in self._sections: return False - - return has_version and has_mode and has_type + return has_version and found_man_mtd == set(self._man_mtd.keys()) class MzTab2(MzTab): @@ -119,8 +122,9 @@ class MzTab2(MzTab): False """ file_ext = "mztab2" - __sections = ["SMH", "SML", "SFH", "SMF", "SEH", "SME", "COM"] - __version_re = r"(2)(\.[0-9])?(\.[0-9])?-M$" + _sections = ["SMH", "SML", "SFH", "SMF", "SEH", "SME", "COM"] + _version_re = r"(2)(\.[0-9])?(\.[0-9])?-M$" + _man_mtd = {"mzTab-ID": None} def __init__(self, **kwd): super(MzTab2, self).__init__(**kwd) @@ -467,14 +471,14 @@ class Edta(TabularData): def _parse_dataline(self, line, tpe): if tpe == 2 or tpe == 3: - l = 4 + idx = 4 else: - l = 3 + idx = 3 try: - line = [float(_) for _ in line[:l]] + line = [float(_) for _ in line[:idx]] except ValueError: return False - if not all(_ >= 0 for _ in line[:l]): + if not all(_ >= 0 for _ in line[:idx]): return False return True From 9a2d2f48c6e7819b3d4a5689a481f1a6f2b0d42b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 20 Feb 2020 11:31:24 +0100 Subject: [PATCH 114/324] MzTab: fix capitalization --- lib/galaxy/datatypes/proteomics.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index ae399561485..fd792e24aea 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -93,7 +93,6 @@ class MzTab(Text): """ Determines whether the file is the correct type. """ has_version = False found_man_mtd = set() - cnt = 0 contents = file_prefix.string_io() for line in contents: if re.match(r"^\s*$", line): @@ -115,10 +114,10 @@ class MzTab2(MzTab): >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('test.mztab2') - >>> mztab2().sniff(fname) + >>> MzTab2().sniff(fname) True >>> fname = get_test_fname('test.mztab') - >>> mztab2().sniff(fname) + >>> MzTab2().sniff(fname) False """ file_ext = "mztab2" From 1111e83ff5c144c0a13661152111528982ba143e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 20 Feb 2020 14:42:11 +0100 Subject: [PATCH 115/324] remove subclass relation between OSW and PQP --- lib/galaxy/datatypes/binary.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 16735eb7731..b93c6d0363f 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1596,10 +1596,10 @@ class PQP(SQlite): table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING'] - return super(PQP, self).sniff(filename) and self.sniff_table_names(filename, table_names) + return super(PQP, self).sniff(filename) and self.sniff_table_names(filename, table_names, True) -class OSW(PQP): +class OSW(SQlite): """ Class describing OpenSwath output @@ -1619,8 +1619,11 @@ class OSW(PQP): def sniff(self, filename): # osw seems to be an extension of pqp (few tables are added) # see also here https://github.com/OpenMS/OpenMS/issues/4365 - table_names = ['FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] - return super(OSW, self).sniff(filename) and self.sniff_table_names(filename, table_names) + table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', + 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', + 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING', + 'FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] + return super(OSW, self).sniff(filename) and self.sniff_table_names(filename, table_names, True) class SQmass(SQlite): From 9e785ba3b86207880f063bf918f38ba7d331eb33 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 20 Feb 2020 14:51:29 +0100 Subject: [PATCH 116/324] PQP: just check that the extra OSW tables are not included --- lib/galaxy/datatypes/binary.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index b93c6d0363f..cde4e1db2f7 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1596,7 +1596,8 @@ class PQP(SQlite): table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING'] - return super(PQP, self).sniff(filename) and self.sniff_table_names(filename, table_names, True) + osw_table_names = ['FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] + return super(PQP, self).sniff(filename) and self.sniff_table_names(filename, table_names) and not self.sniff_table_names(filename, osw_table_names) class OSW(SQlite): @@ -1623,7 +1624,7 @@ class OSW(SQlite): 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING', 'FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] - return super(OSW, self).sniff(filename) and self.sniff_table_names(filename, table_names, True) + return super(OSW, self).sniff(filename) and self.sniff_table_names(filename, table_names) class SQmass(SQlite): From fce58247f60012315df81f2da640527e1c469efc Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 21 Feb 2020 10:58:33 +0100 Subject: [PATCH 117/324] disable sublass="true" for osw, pqp, sqmass --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 6 +++--- lib/galaxy/datatypes/binary.py | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index c6bd174ce25..e7d2da99200 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -358,9 +358,9 @@ - - - + + + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index cde4e1db2f7..7e7c6bc451f 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1593,11 +1593,13 @@ class PQP(SQlite): for now VERSION GENE PEPTIDE_GENE_MAPPING are excluded, since there is test data wo these tables, see also here https://github.com/OpenMS/OpenMS/issues/4365 """ + if not super(PQP, self).sniff(filename): + return False table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING'] osw_table_names = ['FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] - return super(PQP, self).sniff(filename) and self.sniff_table_names(filename, table_names) and not self.sniff_table_names(filename, osw_table_names) + return self.sniff_table_names(filename, table_names) and not self.sniff_table_names(filename, osw_table_names) class OSW(SQlite): @@ -1620,11 +1622,13 @@ class OSW(SQlite): def sniff(self, filename): # osw seems to be an extension of pqp (few tables are added) # see also here https://github.com/OpenMS/OpenMS/issues/4365 + if not super(OSW, self).sniff(filename): + return False table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING', 'FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] - return super(OSW, self).sniff(filename) and self.sniff_table_names(filename, table_names) + return self.sniff_table_names(filename, table_names) class SQmass(SQlite): From 274ad1d770f884ca3dd346613f61356429d30bba Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 21 Feb 2020 11:13:35 +0100 Subject: [PATCH 118/324] remove sqlite data types with sniffers from registry --- lib/galaxy/datatypes/registry.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 67c01e39f57..cfbdd4f8223 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -770,18 +770,15 @@ class Registry(object): 'eland' : tabular.Eland(), 'fastq' : sequence.Fastq(), 'fastqsanger' : sequence.FastqSanger(), - 'gemini.sqlite' : binary.GeminiSQLite(), 'gtf' : interval.Gtf(), 'gff' : interval.Gff(), 'gff3' : interval.Gff3(), 'genetrack' : tracks.GeneTrack(), 'h5' : binary.H5(), - 'idpdb' : binary.IdpDB(), 'interval' : interval.Interval(), 'laj' : images.Laj(), 'lav' : sequence.Lav(), 'maf' : sequence.Maf(), - 'mz.sqlite' : binary.MzSQlite(), 'pileup' : tabular.Pileup(), 'qualsolid' : qualityscore.QualityScoreSOLiD(), 'qualsolexa' : qualityscore.QualityScoreSolexa(), @@ -801,7 +798,6 @@ class Registry(object): 'axt' : 'text/plain', 'bam' : 'application/octet-stream', 'bed' : 'text/plain', - 'blib' : 'application/octet-stream', 'customtrack' : 'text/plain', 'csfasta' : 'text/plain', 'db3' : 'application/octet-stream', @@ -809,18 +805,15 @@ class Registry(object): 'fasta' : 'text/plain', 'fastq' : 'text/plain', 'fastqsanger' : 'text/plain', - 'gemini.sqlite' : 'application/octet-stream', 'gtf' : 'text/plain', 'gff' : 'text/plain', 'gff3' : 'text/plain', 'h5' : 'application/octet-stream', - 'idpdb' : 'application/octet-stream', 'interval' : 'text/plain', 'laj' : 'text/plain', 'lav' : 'text/plain', 'maf' : 'text/plain', 'memexml' : 'application/xml', - 'mz.sqlite' : 'application/octet-stream', 'pileup' : 'text/plain', 'qualsolid' : 'text/plain', 'qualsolexa' : 'text/plain', @@ -846,10 +839,6 @@ class Registry(object): binary.Bam(), binary.Sff(), binary.H5(), - binary.GeminiSQLite(), - binary.MzSQlite(), - binary.IdpDB(), - binary.SQlite(), xml.GenericXml(), sequence.Maf(), sequence.Lav(), From 9d9fa252f2e34428c1e5751d4e96d55e75d81814 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 21 Feb 2020 11:48:29 +0100 Subject: [PATCH 119/324] move SQlite datatype up in the datatypes_config.sample --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index e7d2da99200..d4324893da2 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -266,6 +266,7 @@ + @@ -357,7 +358,6 @@ - From c6d0955ad22613621b9c78aa53bd52d8400746b8 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 21 Feb 2020 12:15:36 +0100 Subject: [PATCH 120/324] remove db3 from registry --- lib/galaxy/datatypes/registry.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index cfbdd4f8223..5b479baf5fb 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -765,7 +765,6 @@ class Registry(object): 'coverage' : coverage.LastzCoverage(), 'customtrack' : interval.CustomTrack(), 'csfasta' : sequence.csFasta(), - 'db3' : binary.SQlite(), 'fasta' : sequence.Fasta(), 'eland' : tabular.Eland(), 'fastq' : sequence.Fastq(), @@ -800,7 +799,6 @@ class Registry(object): 'bed' : 'text/plain', 'customtrack' : 'text/plain', 'csfasta' : 'text/plain', - 'db3' : 'application/octet-stream', 'eland' : 'application/octet-stream', 'fasta' : 'text/plain', 'fastq' : 'text/plain', From 0deaf09ad3143a6eaf0f1c466371e4fbabd53a54 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 21 Feb 2020 20:36:17 +0100 Subject: [PATCH 121/324] match thermo.raw extension as in datatypes_conf --- lib/galaxy/datatypes/proteomics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index fd792e24aea..f3a28933844 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -782,7 +782,7 @@ class ThermoRAW(Binary): """Class describing a Thermo Finnigan binary RAW file""" edam_data = "data_2536" edam_format = "format_3712" - file_ext = "raw" + file_ext = "thermo.raw" def sniff(self, filename): # Thermo Finnigan RAW format is proprietary and hence not well documented. From ec23d175a7e690e9261bfd2f074edff5793bb8bb Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 21 Feb 2020 22:03:09 +0100 Subject: [PATCH 122/324] add sirius.ms datatype as subclass of text --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index d4324893da2..0e5dfcdf2b8 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -248,6 +248,7 @@ + From 68c92b89e73f919d7e6c573ceaa79497586caaec Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sat, 22 Feb 2020 16:02:26 +0100 Subject: [PATCH 123/324] remove duplicate osw, pqp entries in conf --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 0e5dfcdf2b8..151062e82e5 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -359,8 +359,6 @@ - - From e1a49cc678c4489e31583895fb8bd5126ebe8fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 22 Feb 2020 17:57:33 +0100 Subject: [PATCH 124/324] ITs --- interactivetool_climate_notebook.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/interactivetool_climate_notebook.xml b/interactivetool_climate_notebook.xml index f867208bfbe..d31103dfc2b 100644 --- a/interactivetool_climate_notebook.xml +++ b/interactivetool_climate_notebook.xml @@ -23,8 +23,8 @@ #if $input: #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) - get -t hid -i '${input.hid}' && - ln -sf '/import/${input.hid}' './jupyter/data/${cleaned_name}' && + get -t hid -i '${input.hid}' && + ln -sf '/import/${input.hid}' './jupyter/data/${cleaned_name}' && #end if ## change into the directory where the notebooks are located @@ -33,15 +33,15 @@ #if $mode.mode_select == 'scratch': ## copy default notebook - cp '/home/jovyan/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb && + cp '$__tool_directory__/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb && jupyter trust ./ipython_galaxy_notebook.ipynb && jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' #else: #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($mode.ipynb.element_identifier)) - get -t hid -i '${mode.ipynb.hid}' && - ln -sf '/import/${mode.ipynb.hid}' './jupyter/data/${cleaned_name}' && + get -t hid -i '${mode.ipynb.hid}' && + ln -sf '/import/${mode.ipynb.hid}' './data/${cleaned_name}' jupyter trust ./${cleaned_name} && #if $mode.run_it: @@ -51,6 +51,7 @@ #end if cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' #end if + ]]> From 6137f9853722fe1a894fff2dfcb6e59366f1bb79 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sat, 22 Feb 2020 16:27:04 +0100 Subject: [PATCH 125/324] add PEFF data type --- .../config/sample/datatypes_conf.xml.sample | 2 ++ lib/galaxy/datatypes/proteomics.py | 26 +++++++++++++++++++ lib/galaxy/datatypes/test/test.peff | 11 ++++++++ 3 files changed, 39 insertions(+) create mode 100644 lib/galaxy/datatypes/test/test.peff diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 151062e82e5..11d948e64dd 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -302,6 +302,7 @@ + @@ -887,6 +888,7 @@ + diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index f3a28933844..8abd76f90c6 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -7,6 +7,7 @@ import re from galaxy.datatypes import data from galaxy.datatypes.binary import Binary from galaxy.datatypes.data import Text +from galaxy.datatypes.sequence import Sequence from galaxy.datatypes.sniff import build_sniff_from_prefix from galaxy.datatypes.tabular import Tabular, TabularData from galaxy.datatypes.xml import GenericXml @@ -243,6 +244,31 @@ class PSMS(Tabular): return False +@build_sniff_from_prefix +class PEFF(Sequence): + """ + PSI Extended FASTA Format + https://github.com/HUPO-PSI/PEFF + """ + file_ext = "peff" + + def sniff_prefix(self, file_prefix): + """ + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname( 'test.peff' ) + >>> PEFF().sniff( fname ) + True + >>> fname = get_test_fname( 'sequence.fasta' ) + >>> PEFF().sniff( fname ) + False + """ + fh = file_prefix.string_io() + if re.match(r"# PEFF \d+.\d+", fh.readline()): + return True + else: + return False + + class PepXmlReport(Tabular): """pepxml converted to tabular report""" edam_data = "data_2536" diff --git a/lib/galaxy/datatypes/test/test.peff b/lib/galaxy/datatypes/test/test.peff new file mode 100644 index 00000000000..a3dc079de2b --- /dev/null +++ b/lib/galaxy/datatypes/test/test.peff @@ -0,0 +1,11 @@ +# PEFF 1.0 +# // +# DbName=Minimal Test example PEFF_Minimal_Valid.peff +# Prefix=sp +# DbSource=http://www.peptideatlas.org/formats/PEFF/PEFF_Minimal_Valid.peff +# DbVersion=1 +# SequenceType=AA +# NumberOfEntries=1 +# // +>sp:Q9Y2X3 \Length=1 +M From 16c843145fadde8789fe6ddeb3ef15f806f39dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 22 Feb 2020 19:32:28 +0100 Subject: [PATCH 126/324] ITs: fix commandline --- interactivetool_climate_notebook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_climate_notebook.xml b/interactivetool_climate_notebook.xml index d31103dfc2b..096bc0b1ce9 100644 --- a/interactivetool_climate_notebook.xml +++ b/interactivetool_climate_notebook.xml @@ -41,7 +41,7 @@ #else: #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($mode.ipynb.element_identifier)) get -t hid -i '${mode.ipynb.hid}' && - ln -sf '/import/${mode.ipynb.hid}' './data/${cleaned_name}' + ln -sf '/import/${mode.ipynb.hid}' './data/${cleaned_name}' && jupyter trust ./${cleaned_name} && #if $mode.run_it: From 0613727e9f869f9ebc5969939053705ed9f1a399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 22 Feb 2020 20:34:19 +0100 Subject: [PATCH 127/324] ITs: fix an other path issue --- interactivetool_climate_notebook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactivetool_climate_notebook.xml b/interactivetool_climate_notebook.xml index 096bc0b1ce9..2f7c989bdb1 100644 --- a/interactivetool_climate_notebook.xml +++ b/interactivetool_climate_notebook.xml @@ -42,7 +42,7 @@ #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($mode.ipynb.element_identifier)) get -t hid -i '${mode.ipynb.hid}' && ln -sf '/import/${mode.ipynb.hid}' './data/${cleaned_name}' && - jupyter trust ./${cleaned_name} && + jupyter trust ./data/${cleaned_name} && #if $mode.run_it: jupyter nbconvert --to notebook --execute --output ./ipython_galaxy_notebook.ipynb --allow-errors ./*.ipynb && From 38f0e3755c1a7dd8bfc952b9bbd3712bc715d332 Mon Sep 17 00:00:00 2001 From: Anne Fouilloux Date: Sun, 23 Feb 2020 00:03:13 +0100 Subject: [PATCH 128/324] Add panoply version for docker container (#55) ITs: get it working --- interactivetool_panoply.xml | 41 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/interactivetool_panoply.xml b/interactivetool_panoply.xml index 5e84abffe8a..d663ce89b61 100644 --- a/interactivetool_panoply.xml +++ b/interactivetool_panoply.xml @@ -1,34 +1,43 @@ - + interative plotting tool for geo-referenced data + + 4.5.1 + - quay.io/nordicesmhub/docker-panoply + quay.io/nordicesmhub/docker-panoply:@VERSION@ 5800 - + output/version.txt && + cp /config/home/output/* output/ | true && + cd output && + sleep 2 && + for file in *; do mv "\$file" "\${file// /_}"; done && + for file in *; do mv "\$file" "\$file.\${file\#\#*.}"; done ]]> - - - + + + From 1b762945899c2b2972473234947678fc94587b50 Mon Sep 17 00:00:00 2001 From: Andreas Skorczyk Date: Tue, 4 Feb 2020 15:22:51 +0100 Subject: [PATCH 129/324] Allow setting container path for containers `preprocess_volumes` only allowed to set the host path (HOST:MODE), which in some cases might not be enough. With this change it is also possible to set the container path (HOST:CONTAINER:MODE), as described in https://docs.docker.com/storage/bind-mounts/ described --- lib/galaxy/tool_util/deps/container_classes.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index 9970334fdea..0d5f53ed939 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -103,6 +103,8 @@ def preprocess_volumes(volumes_raw_str, container_type): ['/a/b:rw'] >>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) ['/a/b:ro', '/a/b/c:rw'] + >>> preprocess_volumes("/a/b:/a:ro,/a/b/c:/a/b:rw", DOCKER_CONTAINER_TYPE) + ['/a/b:/a:ro', '/a/b/c:/a/b:rw'] >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) ['/a/b:ro', '/a/b/c:rw'] >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE) @@ -115,8 +117,12 @@ def preprocess_volumes(volumes_raw_str, container_type): for volume_raw_str in volumes_raw_strs: volume_parts = volume_raw_str.split(":") - if len(volume_parts) > 2: + if len(volume_parts) > 3: raise Exception("Unparsable volumes string in configuration [%s]" % volumes_raw_str) + if len(volume_parts) == 3: + volume_parts = ["%s:%s" % (volume_parts[0], volume_parts[1]), volume_parts[2]] + if len(volume_parts) == 2 and volume_parts[1] not in ("rw", "ro", "default_ro"): + volume_parts = ["%s:%s" % (volume_parts[0], volume_parts[1]), "rw"] if len(volume_parts) == 1: volume_parts.append("rw") volumes.append(volume_parts) From 3a51247e8b235a0ea2dd53e1673c06002e7ce9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Mon, 9 Mar 2020 08:52:41 +0100 Subject: [PATCH 130/324] Delete interactivetool_hicbrowser.xml --- interactivetool_hicbrowser.xml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 interactivetool_hicbrowser.xml diff --git a/interactivetool_hicbrowser.xml b/interactivetool_hicbrowser.xml deleted file mode 100644 index 7aaa8a3e8e7..00000000000 --- a/interactivetool_hicbrowser.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - bgruening/hicbrowser - - - - 80 - - - - - - - - - - - - - - Visualising HiC data with HiCBrowser. - - From 47a1c793a2b027f8fd4f5485e2a15dd8541abea5 Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 16 Mar 2020 12:08:33 -0400 Subject: [PATCH 131/324] Combine tests for creating and switching histories. Since creating a new history automatically activates it, I think it's sufficient to test switching *back* to the original history after creating a new one. This also adds a way to select the individual history columns in the multi-view and the switch-to button for each history. --- lib/galaxy/selenium/navigation.yml | 5 ++ .../selenium/test_history_multi_view.py | 56 +++++++++---------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index c6160ba379d..a3f326283c9 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -213,6 +213,11 @@ multi_history_view: selector: '//button[contains(text(), "Copy")]' +multiple_histories: + selectors: + _: '#history-column-${history_id}' + switch_button: '${_} button.switch-to' + history_copy_elements: selectors: diff --git a/lib/galaxy_test/selenium/test_history_multi_view.py b/lib/galaxy_test/selenium/test_history_multi_view.py index 6aee9669198..a1d2cda8bcf 100644 --- a/lib/galaxy_test/selenium/test_history_multi_view.py +++ b/lib/galaxy_test/selenium/test_history_multi_view.py @@ -95,40 +95,34 @@ class HistoryMultiViewTestCase(SeleniumTestCase): self.assert_history(history_id, should_exist=False) @selenium_test - def test_switching_history(self): - history_id = self.current_history_id() - method = self.dataset_collection_populator.create_list_in_history(history_id, contents=["0", "1", "0", "1"]).json - - self.prepare_multi_history_view(method) - self.copy_history(history_id) - self.components.multi_history_view.switch_history.wait_for_and_click() - self.sleep_for(self.wait_types.UX_RENDER) - self.assert_history(history_id, histories_number=2) - # assert that id of current history equals history_id - assert self.components.multi_history_view.current_history_check(history_id=history_id).is_displayed - self.components.multi_history_view.switch_history.wait_for_and_click() - self.sleep_for(self.wait_types.UX_RENDER) - # assert that 'history_id' is not current history - self.assertRaises(NoSuchElementException, lambda: - self.components.multi_history_view.current_history_check(history_id=history_id).is_displayed) - - @selenium_test - def test_create_new_history(self): - history_id = self.current_history_id() - method = self.dataset_collection_populator.create_list_in_history(history_id, contents=["0", "1", "0", "1"]).json - - self.prepare_multi_history_view(method) - # assert that empty history is not created in advance - self.components.multi_history_view.empty_message_check.assert_absent_or_hidden() - + def test_switch_history(self): + ''' + 1. Load the multi history view. There should be a selector for the button + to create a new history. + 2. Create a new history. This *should* automatically switch to the newly + created history. + 3. Switch back to the original history. A button should appear on the old, + previously created history that allows switching back to that one, and + the history ID should now match the ID of the history with which we + started. + ''' + self.home() + original_history_id = self.current_history_id() + # Load the multi-view + self.components.history_panel.multi_view_button.wait_for_and_click() + # Creating a new history should automatically switch to it self.components.multi_history_view.create_new_button.wait_for_and_click() self.sleep_for(self.wait_types.UX_RENDER) - - self.assert_history(history_id, histories_number=2) + new_history_id = self.current_history_id() + # Otherwise this assertion would fail + self.screenshot("multi_history_switch_created_history") + self.assertNotEqual(original_history_id, new_history_id) + # Switch back to the original history + switch_button = self.components.multiple_histories._(history_id=original_history_id).switch_button + switch_button.wait_for_and_click() self.sleep_for(self.wait_types.UX_RENDER) - - # assert that empty history is present - self.components.multi_history_view.empty_message_check.wait_for_present() + self.screenshot("multi_history_switch_changed_history") + self.assertEqual(original_history_id, self.current_history_id()) def assert_history(self, history_id, histories_number=1, should_exist=True): histories = self.components.multi_history_view.histories.all() From cbc606023032c94aed76fbfdd75392cbaacfcd18 Mon Sep 17 00:00:00 2001 From: Dave B Date: Tue, 17 Mar 2020 13:00:23 -0400 Subject: [PATCH 132/324] Cleaner selectors for switch history button. --- lib/galaxy/selenium/navigation.yml | 7 +------ lib/galaxy_test/selenium/test_history_multi_view.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index a3f326283c9..1ec1f72e80b 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -198,7 +198,7 @@ multi_history_view: delete: '.copy-history' current_history_check: '#history-column-${history_id} .current-label' empty_message_check: '.empty-message' - + switch_button: '#history-column-${history_id} .switch-to' history_dropdown_menu: selectors: @@ -213,11 +213,6 @@ multi_history_view: selector: '//button[contains(text(), "Copy")]' -multiple_histories: - selectors: - _: '#history-column-${history_id}' - switch_button: '${_} button.switch-to' - history_copy_elements: selectors: diff --git a/lib/galaxy_test/selenium/test_history_multi_view.py b/lib/galaxy_test/selenium/test_history_multi_view.py index a1d2cda8bcf..65123c24417 100644 --- a/lib/galaxy_test/selenium/test_history_multi_view.py +++ b/lib/galaxy_test/selenium/test_history_multi_view.py @@ -118,7 +118,7 @@ class HistoryMultiViewTestCase(SeleniumTestCase): self.screenshot("multi_history_switch_created_history") self.assertNotEqual(original_history_id, new_history_id) # Switch back to the original history - switch_button = self.components.multiple_histories._(history_id=original_history_id).switch_button + switch_button = self.components.multi_history_view.switch_button(history_id=original_history_id) switch_button.wait_for_and_click() self.sleep_for(self.wait_types.UX_RENDER) self.screenshot("multi_history_switch_changed_history") From 24c34afe8eed304e64aeb943d7e6d4410fb153d2 Mon Sep 17 00:00:00 2001 From: Dave B Date: Wed, 18 Mar 2020 12:37:37 -0400 Subject: [PATCH 133/324] Add assertion for history count. --- lib/galaxy_test/selenium/test_history_multi_view.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy_test/selenium/test_history_multi_view.py b/lib/galaxy_test/selenium/test_history_multi_view.py index 65123c24417..a80cb26b07c 100644 --- a/lib/galaxy_test/selenium/test_history_multi_view.py +++ b/lib/galaxy_test/selenium/test_history_multi_view.py @@ -110,6 +110,8 @@ class HistoryMultiViewTestCase(SeleniumTestCase): original_history_id = self.current_history_id() # Load the multi-view self.components.history_panel.multi_view_button.wait_for_and_click() + # There should be only one + self.assert_history(original_history_id, histories_number=1) # Creating a new history should automatically switch to it self.components.multi_history_view.create_new_button.wait_for_and_click() self.sleep_for(self.wait_types.UX_RENDER) From 44ad8910f1b4f3d64daa4ef8b1cc9fc6d49f2254 Mon Sep 17 00:00:00 2001 From: Peter Selten Date: Wed, 18 Mar 2020 23:33:40 +0100 Subject: [PATCH 134/324] Update social-auth-core to version 3.3.0 and allow no secondary auth to be set if the provider isn't google --- lib/galaxy/authnz/psa_authnz.py | 6 ++++-- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index 0ef26891ff0..b8b3388dfd5 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -116,8 +116,10 @@ class PSAAuthnz(IdentityProvider): # Secondary AuthZ with Google identities is currently supported if provider != "google": - del self.config["SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER"] - del self.config["SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT"] + if "SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER" in self.config: + del self.config["SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER"] + if "SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT" in self.config: + del self.config["SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT"] def _setup_idp(self, oidc_backend_config): self.config[setting_name('AUTH_EXTRA_ARGUMENTS')] = {'access_type': 'offline'} diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 0d47bdffdc2..a2deae34b45 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -170,7 +170,7 @@ setuptools-scm==3.5.0 shellescape==3.4.1 simplejson==3.17.0 six==1.11.0 -social-auth-core[openidconnect]==3.1.0+gx0 +social-auth-core[openidconnect]==3.3.0 sqlalchemy-migrate==0.13.0 sqlalchemy-utils==0.36.1 sqlalchemy==1.3.13 From c46524a198ffb1a39bd30443766ab74346cca032 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 19 Mar 2020 15:56:23 +0100 Subject: [PATCH 135/324] wait for histories to be present --- lib/galaxy_test/selenium/test_history_multi_view.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy_test/selenium/test_history_multi_view.py b/lib/galaxy_test/selenium/test_history_multi_view.py index a80cb26b07c..14b59d137ee 100644 --- a/lib/galaxy_test/selenium/test_history_multi_view.py +++ b/lib/galaxy_test/selenium/test_history_multi_view.py @@ -127,6 +127,7 @@ class HistoryMultiViewTestCase(SeleniumTestCase): self.assertEqual(original_history_id, self.current_history_id()) def assert_history(self, history_id, histories_number=1, should_exist=True): + self.components.multi_history_view.histories.wait_for_present() histories = self.components.multi_history_view.histories.all() assert len(histories) == histories_number # search for history with history_id From 5ad5939615fe5c3e708b353a168172df786b861e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2020 21:54:52 +0000 Subject: [PATCH 136/324] Bump tensorflow from 1.12.2 to 1.15.2 in /lib/galaxy/dependencies Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 1.12.2 to 1.15.2. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v1.12.2...v1.15.2) Signed-off-by: dependabot[bot] --- lib/galaxy/dependencies/conditional-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index b1f96df9525..44f61faf1a8 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -31,4 +31,4 @@ influxdb # Deep learning packages for tool recommendation keras==2.2.4 -tensorflow==1.12.2 +tensorflow==1.15.2 From 14a52c20fc7a72758a3e5c17d38713badb699db0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 7 Feb 2020 15:31:53 -0500 Subject: [PATCH 137/324] Refactor toward testing workflow collection outputs... ... by Planemo via tool_util. --- lib/galaxy/tool_util/cwl/util.py | 28 ++++-- lib/galaxy/tool_util/verify/interactor.py | 103 ++++++++++++---------- 2 files changed, 78 insertions(+), 53 deletions(-) diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index 29c6f0aa9d5..3892b6aeeba 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -354,17 +354,17 @@ class DirectoryUploadTarget(object): return "DirectoryUploadTarget[tar_path=%s]" % self.tar_path -GalaxyOutput = namedtuple("GalaxyOutput", ["history_id", "history_content_type", "history_content_id"]) +GalaxyOutput = namedtuple("GalaxyOutput", ["history_id", "history_content_type", "history_content_id", "metadata"]) def tool_response_to_output(tool_response, history_id, output_id): for output in tool_response["outputs"]: if output["output_name"] == output_id: - return GalaxyOutput(history_id, "dataset", output["id"]) + return GalaxyOutput(history_id, "dataset", output["id"], None) for output_collection in tool_response["output_collections"]: if output_collection["output_name"] == output_id: - return GalaxyOutput(history_id, "dataset_collection", output_collection["id"]) + return GalaxyOutput(history_id, "dataset_collection", output_collection["id"], None) raise Exception("Failed to find output with label [%s]" % output_id) @@ -372,10 +372,10 @@ def tool_response_to_output(tool_response, history_id, output_id): def invocation_to_output(invocation, history_id, output_id): if output_id in invocation["outputs"]: dataset = invocation["outputs"][output_id] - galaxy_output = GalaxyOutput(history_id, "dataset", dataset["id"]) + galaxy_output = GalaxyOutput(history_id, "dataset", dataset["id"], None) elif output_id in invocation["output_collections"]: collection = invocation["output_collections"][output_id] - galaxy_output = GalaxyOutput(history_id, "dataset_collection", collection["id"]) + galaxy_output = GalaxyOutput(history_id, "dataset_collection", collection["id"], None) else: raise Exception("Failed to find output with label [%s] in [%s]" % (output_id, invocation)) @@ -391,14 +391,26 @@ def output_to_cwl_json( interface via Galaxy. """ def element_to_cwl_json(element): + object = element["object"] + content_type = object.get("history_content_type") + metadata = None + if content_type is None: + content_type = "dataset_collection" + metadata = element["object"] + metadata["history_content_type"] = content_type element_output = GalaxyOutput( galaxy_output.history_id, - element["object"]["history_content_type"], - element["object"]["id"], + content_type, + object["id"], + metadata, ) return output_to_cwl_json(element_output, get_metadata, get_dataset, get_extra_files, pseduo_location=pseduo_location) - output_metadata = get_metadata(galaxy_output.history_content_type, galaxy_output.history_content_id) + output_metadata = galaxy_output.metadata + if output_metadata is None: + output_metadata = get_metadata(galaxy_output.history_content_type, galaxy_output.history_content_id) + + print(output_metadata) def dataset_dict_to_json_content(dataset_dict): if "content" in dataset_dict: diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index 2f6cfb5c4bd..792448d9369 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -147,54 +147,19 @@ class GalaxyInteractorApi(object): return response.json() def verify_output_collection(self, output_collection_def, output_collection_id, history, tool_id): - name = output_collection_def.name data_collection = self._get("dataset_collections/%s" % output_collection_id, data={"instance_type": "history"}).json() - def get_element(elements, id): - for element in elements: - if element["element_identifier"] == id: - return element - return False + def verify_dataset(element, element_attrib, element_outfile): + hda = element["object"] + self.verify_output_dataset( + history, + hda_id=hda["id"], + outfile=element_outfile, + attributes=element_attrib, + tool_id=tool_id + ) - expected_collection_type = output_collection_def.collection_type - if expected_collection_type: - collection_type = data_collection["collection_type"] - if expected_collection_type != collection_type: - template = "Expected output collection [%s] to be of type [%s], was of type [%s]." - message = template % (name, expected_collection_type, collection_type) - raise AssertionError(message) - - expected_element_count = output_collection_def.count - if expected_element_count: - actual_element_count = len(data_collection["elements"]) - if expected_element_count != actual_element_count: - template = "Expected output collection [%s] to have %s elements, but it had %s." - message = template % (name, expected_element_count, actual_element_count) - raise AssertionError(message) - - def verify_elements(element_objects, element_tests): - for element_identifier, (element_outfile, element_attrib) in element_tests.items(): - element = get_element(element_objects, element_identifier) - if not element: - template = "Failed to find identifier [%s] for testing, tool generated collection elements [%s]" - message = template % (element_identifier, element_objects) - raise AssertionError(message) - - element_type = element["element_type"] - if element_type != "dataset_collection": - hda = element["object"] - self.verify_output_dataset( - history, - hda_id=hda["id"], - outfile=element_outfile, - attributes=element_attrib, - tool_id=tool_id - ) - if element_type == "dataset_collection": - elements = element["object"]["elements"] - verify_elements(elements, element_attrib.get("elements", {})) - - verify_elements(data_collection["elements"], output_collection_def.element_tests) + verify_collection(output_collection_def, data_collection, verify_dataset) def verify_output(self, history_id, jobs, output_data, output_testdef, tool_id, maxseconds): outfile = output_testdef.outfile @@ -724,6 +689,54 @@ def verify_hid(filename, hda_id, attributes, test_data_downloader, hid="", datas ) +def verify_collection(output_collection_def, data_collection, verify_dataset): + name = output_collection_def.name + + def get_element(elements, id): + for element in elements: + if element["element_identifier"] == id: + return element + return False + + expected_collection_type = output_collection_def.collection_type + if expected_collection_type: + collection_type = data_collection["collection_type"] + if expected_collection_type != collection_type: + template = "Expected output collection [%s] to be of type [%s], was of type [%s]." + message = template % (name, expected_collection_type, collection_type) + raise AssertionError(message) + + expected_element_count = output_collection_def.count + if expected_element_count: + actual_element_count = len(data_collection["elements"]) + if expected_element_count != actual_element_count: + template = "Expected output collection [%s] to have %s elements, but it had %s." + message = template % (name, expected_element_count, actual_element_count) + raise AssertionError(message) + + def verify_elements(element_objects, element_tests): + for element_identifier, element_test in element_tests.items(): + if isinstance(element_test, dict): + element_outfile, element_attrib = None, element_test + else: + element_outfile, element_attrib = element_test + + element = get_element(element_objects, element_identifier) + if not element: + template = "Failed to find identifier [%s] for testing, tool generated collection elements [%s]" + message = template % (element_identifier, element_objects) + raise AssertionError(message) + + element_type = element["element_type"] + if element_type != "dataset_collection": + verify_dataset(element, element_attrib, element_outfile) + if element_type == "dataset_collection": + elements = element["object"]["elements"] + verify_elements(elements, element_attrib.get("elements", {})) + + verify_elements(data_collection["elements"], output_collection_def.element_tests) + + def _verify_composite_datatype_file_content(file_name, hda_id, base_name=None, attributes=None, dataset_fetcher=None, test_data_downloader=None, keep_outputs_dir=False, mode='file'): assert dataset_fetcher is not None From a22e35eea5a81449eb5d614c1949a50400867618 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 7 Feb 2020 16:50:54 -0500 Subject: [PATCH 138/324] Spelling fix. --- lib/galaxy/tool_util/verify/interactor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index 792448d9369..d9c3b29c99d 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -167,7 +167,7 @@ class GalaxyInteractorApi(object): name = output_testdef.name self.wait_for_jobs(history_id, jobs, maxseconds) hid = self.__output_id(output_data) - # TODO: Twill version verifys dataset is 'ok' in here. + # TODO: Twill version verifies dataset is 'ok' in here. self.verify_output_dataset(history_id=history_id, hda_id=hid, outfile=outfile, attributes=attributes, tool_id=tool_id) primary_datasets = attributes.get('primary_datasets', {}) From c9ca69075c6500a34d05c9691c781995b3740157 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 20 Mar 2020 18:17:54 +0100 Subject: [PATCH 139/324] fix documentation for data checksum attribute the separator seems to be `$`: https://github.com/galaxyproject/galaxy/blob/519e35e9682a3d8ce8de0f3736b97e84fba583c4/lib/galaxy/tool_util/verify/__init__.py#L76 --- lib/galaxy/tool_util/xsd/galaxy.xsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index a812c6dc04d..e074d8811aa 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -1291,8 +1291,8 @@ and this can be used instead. From 2ae3ed1c5a33f340c9b07a5558d8c820b6bbe88b Mon Sep 17 00:00:00 2001 From: Juan David Mendez Solano Date: Fri, 20 Mar 2020 21:42:25 +0100 Subject: [PATCH 140/324] Implementation of Selenium test for signout from user preferences --- lib/galaxy/selenium/navigation.yml | 1 + lib/galaxy_test/selenium/test_sign_out.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 lib/galaxy_test/selenium/test_sign_out.py diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index f35dab23a0b..50f81ed5016 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -70,6 +70,7 @@ masthead: preferences: selectors: + sign_out: "#edit-preferences-custom-builds" change_password: "#edit-preferences-password" manage_information: '#edit-preferences-information' current_email: "#user-preferences-current-email" diff --git a/lib/galaxy_test/selenium/test_sign_out.py b/lib/galaxy_test/selenium/test_sign_out.py new file mode 100644 index 00000000000..9a160bef9ac --- /dev/null +++ b/lib/galaxy_test/selenium/test_sign_out.py @@ -0,0 +1,22 @@ +from .framework import ( + selenium_test, + SeleniumTestCase +) + + +class SignOutTestCase(SeleniumTestCase): + @selenium_test + def test_sign_out(self): + email = self._get_random_email() + self.register(email) + self.click_masthead_user() + self.components.masthead.preferences.wait_for_and_click() + self.components.preferences.sign_out.wait_for_and_click() + cancel_button = self.driver.find_element_by_xpath("//button[@id='button-0']") + cancel_button.click() + assert self.is_logged_in() + new_email = self.driver.find_element_by_id("user-preferences-current-email").text + self.assertTrue(email == new_email) + signout_button = self.driver.find_element_by_xpath("//button[@id='button-1']") + signout_button.click() + assert not self.is_logged_in() From 9d4f6ab0cde15e3dd5106568496d7b0590e0a1cf Mon Sep 17 00:00:00 2001 From: Juan David Mendez Solano Date: Fri, 20 Mar 2020 22:57:57 +0100 Subject: [PATCH 141/324] Change id link id from custome-builds into sign-out --- client/galaxy/scripts/components/User/UserPreferencesModel.js | 2 +- lib/galaxy/selenium/navigation.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/components/User/UserPreferencesModel.js b/client/galaxy/scripts/components/User/UserPreferencesModel.js index 702d038ae39..847c0b226c8 100644 --- a/client/galaxy/scripts/components/User/UserPreferencesModel.js +++ b/client/galaxy/scripts/components/User/UserPreferencesModel.js @@ -83,7 +83,7 @@ export const getUserPreferencesModel = () => { }, logout: { title: _l("Sign Out"), - id: "edit-preferences-custom-builds", + id: "edit-preferences-sign-out", description: _l("Click here to sign out of all sessions."), icon: "fa-sign-out", shouldRender: !!Galaxy.session_csrf_token diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index 50f81ed5016..9e4b125817f 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -70,7 +70,7 @@ masthead: preferences: selectors: - sign_out: "#edit-preferences-custom-builds" + sign_out: "#edit-preferences-sign-out" change_password: "#edit-preferences-password" manage_information: '#edit-preferences-information' current_email: "#user-preferences-current-email" From 0b5356888130fde5fd9edcc4385699fb906827dc Mon Sep 17 00:00:00 2001 From: "jalili.vahid@gmail.com" Date: Mon, 23 Mar 2020 13:39:50 -0700 Subject: [PATCH 142/324] Separate ObjectStore interface and base implementation. # - Interface name follows naming convention (with I prefix); # - Interface does not implement any methods; # - Base provides common methods for all the derived types. --- lib/galaxy/objectstore/__init__.py | 85 ++++++++++++++++-------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index b1dccf8b249..38000337181 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -38,9 +38,9 @@ NO_SESSION_ERROR_MESSAGE = "Attempted to 'create' object store entity in configu log = logging.getLogger(__name__) -class ObjectStore(object): +class IObjectStore(object): - """ObjectStore abstract interface. + """ObjectStore interface. FIELD DESCRIPTIONS (these apply to all the methods in this class): @@ -82,47 +82,10 @@ class ObjectStore(object): 000/obj.id) """ - def __init__(self, config, config_dict=None, **kwargs): - """ - :type config: object - :param config: An object, most likely populated from - `galaxy/config.ini`, having the following attributes: - - * object_store_check_old_style (only used by the - :class:`DiskObjectStore` subclass) - * jobs_directory -- Each job is given a unique empty directory - as its current working directory. This option defines in what - parent directory those directories will be created. - * new_file_path -- Used to set the 'temp' extra_dir. - """ - if config_dict is None: - config_dict = {} - self.running = True - self.config = config - self.check_old_style = config.object_store_check_old_style - extra_dirs = {} - extra_dirs['job_work'] = config.jobs_directory - extra_dirs['temp'] = config.new_file_path - extra_dirs.update(dict( - (e['type'], e['path']) for e in config_dict.get('extra_dirs', []))) - self.extra_dirs = extra_dirs - - def shutdown(self): - """Close any connections for this ObjectStore.""" - self.running = False - def exists(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None): """Return True if the object identified by `obj` exists, False otherwise.""" raise NotImplementedError() - def file_ready(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): - """ - Check if a file corresponding to a dataset is ready to be used. - - Return True if so, False otherwise - """ - return True - def create(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Mark the object (`obj`) as existing in the store, but with no content. @@ -219,6 +182,46 @@ class ObjectStore(object): """ raise NotImplementedError() + +class BaseObjectStore(IObjectStore): + + def __init__(self, config, config_dict=None, **kwargs): + """ + :type config: object + :param config: An object, most likely populated from + `galaxy/config.ini`, having the following attributes: + + * object_store_check_old_style (only used by the + :class:`DiskObjectStore` subclass) + * jobs_directory -- Each job is given a unique empty directory + as its current working directory. This option defines in what + parent directory those directories will be created. + * new_file_path -- Used to set the 'temp' extra_dir. + """ + if config_dict is None: + config_dict = {} + self.running = True + self.config = config + self.check_old_style = config.object_store_check_old_style + extra_dirs = {} + extra_dirs['job_work'] = config.jobs_directory + extra_dirs['temp'] = config.new_file_path + extra_dirs.update(dict( + (e['type'], e['path']) for e in config_dict.get('extra_dirs', []))) + self.extra_dirs = extra_dirs + + def shutdown(self): + """Close any connections for this ObjectStore.""" + self.running = False + + def file_ready(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + """ + Check if a file corresponding to a dataset is ready to be used. + + Return True if so, False otherwise + """ + return True + @classmethod def parse_xml(clazz, config_xml): """Parse an XML description of a configuration for this object store. @@ -252,7 +255,7 @@ class ObjectStore(object): return obj.id -class ConcreteObjectStore(ObjectStore): +class ConcreteObjectStore(BaseObjectStore): """Subclass of ObjectStore for stores that don't delegate (non-nested). Currently only adds store_by functionality. Which doesn't make @@ -548,7 +551,7 @@ class DiskObjectStore(ConcreteObjectStore): return (float(st.f_blocks - st.f_bavail) / st.f_blocks) * 100 -class NestedObjectStore(ObjectStore): +class NestedObjectStore(BaseObjectStore): """ Base for ObjectStores that use other ObjectStores. From d4abc786c74fe17d092c6039a097c0ab17b603e6 Mon Sep 17 00:00:00 2001 From: "jalili.vahid@gmail.com" Date: Mon, 23 Mar 2020 18:53:49 -0700 Subject: [PATCH 143/324] Shadow interface methods in the BaseObjectStore. --- lib/galaxy/objectstore/__init__.py | 132 +++++++++++++++++---------- lib/galaxy/objectstore/azure_blob.py | 38 ++++---- lib/galaxy/objectstore/cloud.py | 38 ++++---- lib/galaxy/objectstore/pithos.py | 36 ++++---- lib/galaxy/objectstore/pulsar.py | 24 ++--- lib/galaxy/objectstore/rods.py | 26 +++--- lib/galaxy/objectstore/s3.py | 38 ++++---- 7 files changed, 184 insertions(+), 148 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 38000337181..f826a91c701 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -254,6 +254,42 @@ class BaseObjectStore(IObjectStore): # job working directories. return obj.id + def _invoke(self, delegate, obj, **kwargs): + return self.__getattribute__("_" + delegate)(obj, **kwargs) + + def exists(self, obj, **kwargs): + return self._invoke('exists', obj, **kwargs) + + def create(self, obj, **kwargs): + return self._invoke('create', obj, **kwargs) + + def empty(self, obj, **kwargs): + return self._invoke('empty', obj, **kwargs) + + def size(self, obj, **kwargs): + return self._invoke('size', obj, **kwargs) + + def delete(self, obj, **kwargs): + return self._invoke('delete', obj, **kwargs) + + def get_data(self, obj, **kwargs): + return self._invoke('get_data', obj, **kwargs) + + def get_filename(self, obj, **kwargs): + return self._invoke('get_filename', obj, **kwargs) + + def update_from_file(self, obj, **kwargs): + return self._invoke('update_from_file', obj, **kwargs) + + def get_object_url(self, obj, **kwargs): + return self._invoke('get_object_url', obj, **kwargs) + + def get_store_usage_percent(self, **kwargs): + return self._invoke('get_store_usage_percent', **kwargs) + + def get_store_by(self, obj, **kwargs): + return self._invoke('get_store_by', obj, **kwargs) + class ConcreteObjectStore(BaseObjectStore): """Subclass of ObjectStore for stores that don't delegate (non-nested). @@ -283,7 +319,7 @@ class ConcreteObjectStore(BaseObjectStore): rval["store_by"] = self.store_by return rval - def get_store_by(self, obj): + def _get_store_by(self, obj): return self.store_by @@ -346,7 +382,7 @@ class DiskObjectStore(ConcreteObjectStore): as_dict["files_dir"] = self.file_path return as_dict - def _get_filename(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + def __get_filename(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Return the absolute path for the file corresponding to the `obj.id`. @@ -428,7 +464,7 @@ class DiskObjectStore(ConcreteObjectStore): path = os.path.join(path, alt_name if alt_name else "dataset_%s.dat" % obj_id) return os.path.abspath(path) - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): """Override `ObjectStore`'s stub and check on disk.""" if self.check_old_style: path = self._construct_path(obj, old_style=True, **kwargs) @@ -438,9 +474,9 @@ class DiskObjectStore(ConcreteObjectStore): return True return os.path.exists(self._construct_path(obj, **kwargs)) - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """Override `ObjectStore`'s stub by creating any files and folders on disk.""" - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): path = self._construct_path(obj, **kwargs) dir_only = kwargs.get('dir_only', False) # Create directory if it does not exist @@ -451,18 +487,18 @@ class DiskObjectStore(ConcreteObjectStore): open(path, 'w').close() # Should be rb? umask_fix_perms(path, self.config.umask, 0o666) - def empty(self, obj, **kwargs): + def _empty(self, obj, **kwargs): """Override `ObjectStore`'s stub by checking file size on disk.""" return self.size(obj, **kwargs) == 0 - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): """Override `ObjectStore`'s stub by return file size on disk. Returns 0 if the object doesn't exist yet or other error. """ - if self.exists(obj, **kwargs): + if self._exists(obj, **kwargs): try: - filepath = self.get_filename(obj, **kwargs) + filepath = self._get_filename(obj, **kwargs) for _ in range(0, 2): size = os.path.getsize(filepath) if size != 0: @@ -475,31 +511,31 @@ class DiskObjectStore(ConcreteObjectStore): else: return 0 - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): """Override `ObjectStore`'s stub; delete the file or folder on disk.""" - path = self.get_filename(obj, **kwargs) + path = self._get_filename(obj, **kwargs) extra_dir = kwargs.get('extra_dir', None) obj_dir = kwargs.get('obj_dir', False) try: if entire_dir and (extra_dir or obj_dir): shutil.rmtree(path) return True - if self.exists(obj, **kwargs): + if self._exists(obj, **kwargs): os.remove(path) return True except OSError as ex: - log.critical('%s delete error %s' % (self._get_filename(obj, **kwargs), ex)) + log.critical('%s delete error %s' % (self.__get_filename(obj, **kwargs), ex)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): """Override `ObjectStore`'s stub; retrieve data directly from disk.""" - data_file = open(self.get_filename(obj, **kwargs), 'r') # Should be rb? + data_file = open(self._get_filename(obj, **kwargs), 'r') # Should be rb? data_file.seek(start) content = data_file.read(count) data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): """ Override `ObjectStore`'s stub. @@ -517,27 +553,27 @@ class DiskObjectStore(ConcreteObjectStore): raise ObjectNotFound return path - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): """`create` parameter is not used in this implementation.""" preserve_symlinks = kwargs.pop('preserve_symlinks', False) # FIXME: symlinks and the object store model may not play well together # these should be handled better, e.g. registering the symlink'd file # as an object if create: - self.create(obj, **kwargs) - if file_name and self.exists(obj, **kwargs): + self._create(obj, **kwargs) + if file_name and self._exists(obj, **kwargs): try: if preserve_symlinks and os.path.islink(file_name): - force_symlink(os.readlink(file_name), self.get_filename(obj, **kwargs)) + force_symlink(os.readlink(file_name), self._get_filename(obj, **kwargs)) else: - path = self.get_filename(obj, **kwargs) + path = self._get_filename(obj, **kwargs) shutil.copy(file_name, path) umask_fix_perms(path, self.config.umask, 0o666) except IOError as ex: - log.critical('Error copying %s to %s: %s' % (file_name, self._get_filename(obj, **kwargs), ex)) + log.critical('Error copying %s to %s: %s' % (file_name, self.__get_filename(obj, **kwargs), ex)) raise ex - def get_object_url(self, obj, **kwargs): + def _get_object_url(self, obj, **kwargs): """ Override `ObjectStore`'s stub. @@ -545,7 +581,7 @@ class DiskObjectStore(ConcreteObjectStore): """ return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): """Override `ObjectStore`'s stub by return percent storage used.""" st = os.statvfs(self.file_path) return (float(st.f_blocks - st.f_bavail) / st.f_blocks) * 100 @@ -570,51 +606,51 @@ class NestedObjectStore(BaseObjectStore): store.shutdown() super(NestedObjectStore, self).shutdown() - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): """Determine if the `obj` exists in any of the backends.""" - return self._call_method('exists', obj, False, False, **kwargs) + return self._call_method('_exists', obj, False, False, **kwargs) def file_ready(self, obj, **kwargs): """Determine if the file for `obj` is ready to be used by any of the backends.""" return self._call_method('file_ready', obj, False, False, **kwargs) - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """Create a backing file in a random backend.""" random.choice(list(self.backends.values())).create(obj, **kwargs) - def empty(self, obj, **kwargs): + def _empty(self, obj, **kwargs): """For the first backend that has this `obj`, determine if it is empty.""" - return self._call_method('empty', obj, True, False, **kwargs) + return self._call_method('_empty', obj, True, False, **kwargs) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): """For the first backend that has this `obj`, return its size.""" - return self._call_method('size', obj, 0, False, **kwargs) + return self._call_method('_size', obj, 0, False, **kwargs) - def delete(self, obj, **kwargs): + def _delete(self, obj, **kwargs): """For the first backend that has this `obj`, delete it.""" - return self._call_method('delete', obj, False, False, **kwargs) + return self._call_method('_delete', obj, False, False, **kwargs) - def get_data(self, obj, **kwargs): + def _get_data(self, obj, **kwargs): """For the first backend that has this `obj`, get data from it.""" - return self._call_method('get_data', obj, ObjectNotFound, True, **kwargs) + return self._call_method('_get_data', obj, ObjectNotFound, True, **kwargs) - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): """For the first backend that has this `obj`, get its filename.""" - return self._call_method('get_filename', obj, ObjectNotFound, True, **kwargs) + return self._call_method('_get_filename', obj, ObjectNotFound, True, **kwargs) - def update_from_file(self, obj, **kwargs): + def _update_from_file(self, obj, **kwargs): """For the first backend that has this `obj`, update it from the given file.""" if kwargs.get('create', False): - self.create(obj, **kwargs) + self._create(obj, **kwargs) kwargs['create'] = False - return self._call_method('update_from_file', obj, ObjectNotFound, True, **kwargs) + return self._call_method('_update_from_file', obj, ObjectNotFound, True, **kwargs) - def get_object_url(self, obj, **kwargs): + def _get_object_url(self, obj, **kwargs): """For the first backend that has this `obj`, get its URL.""" - return self._call_method('get_object_url', obj, None, False, **kwargs) + return self._call_method('_get_object_url', obj, None, False, **kwargs) - def get_store_by(self, obj): - return self._call_method('get_store_by', obj, None, False) + def _get_store_by(self, obj): + return self._call_method('_get_store_by', obj, None, False) def _repr_object_for_exception(self, obj): try: @@ -794,9 +830,9 @@ class DistributedObjectStore(NestedObjectStore): self.weighted_backend_ids = new_weighted_backend_ids self.sleeper.sleep(120) # Test free space every 2 minutes - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """The only method in which obj.object_store_id may be None.""" - if obj.object_store_id is None or not self.exists(obj, **kwargs): + if obj.object_store_id is None or not self._exists(obj, **kwargs): if obj.object_store_id is None or obj.object_store_id not in self.backends: try: obj.object_store_id = random.choice(self.weighted_backend_ids) @@ -883,14 +919,14 @@ class HierarchicalObjectStore(NestedObjectStore): as_dict["backends"] = backends return as_dict - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): """Check all child object stores.""" for store in self.backends.values(): if store.exists(obj, **kwargs): return True return False - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """Call the primary object store.""" self.backends[0].create(obj, **kwargs) diff --git a/lib/galaxy/objectstore/azure_blob.py b/lib/galaxy/objectstore/azure_blob.py index 93c8c335d8c..b47c152edb2 100644 --- a/lib/galaxy/objectstore/azure_blob.py +++ b/lib/galaxy/objectstore/azure_blob.py @@ -310,7 +310,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): # Public Methods # ################## - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): in_cache = in_azure = False rel_path = self._construct_path(obj, **kwargs) @@ -357,9 +357,9 @@ class AzureBlobObjectStore(ConcreteObjectStore): return False - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): # Pull out locally used fields extra_dir = kwargs.get('extra_dir', None) @@ -393,25 +393,25 @@ class AzureBlobObjectStore(ConcreteObjectStore): open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_os(rel_path, from_string='') - def empty(self, obj, **kwargs): - if self.exists(obj, **kwargs): - return bool(self.size(obj, **kwargs) > 0) + def _empty(self, obj, **kwargs): + if self._exists(obj, **kwargs): + return bool(self._size(obj, **kwargs) > 0) else: raise ObjectNotFound('objectstore.empty, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) if self._in_cache(rel_path): try: return os.path.getsize(self._get_cache_path(rel_path)) except OSError as ex: log.info("Could not get size of file '%s' in local cache, will try Azure. Error: %s", rel_path, ex) - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): return self._get_size_in_azure(rel_path) log.warning("Did not find dataset '%s', returning 0 for size", rel_path) return 0 - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): rel_path = self._construct_path(obj, **kwargs) extra_dir = kwargs.get('extra_dir', None) base_dir = kwargs.get('base_dir', None) @@ -445,10 +445,10 @@ class AzureBlobObjectStore(ConcreteObjectStore): except AzureHttpError: log.exception("Could not delete blob '%s' from Azure", rel_path) except OSError: - log.exception('%s delete error', self.get_filename(obj, **kwargs)) + log.exception('%s delete error', self._get_filename(obj, **kwargs)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Check cache first and get file if not there if not self._in_cache(rel_path): @@ -460,7 +460,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) base_dir = kwargs.get('base_dir', None) dir_only = kwargs.get('dir_only', False) @@ -483,7 +483,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): if self._in_cache(rel_path): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): if dir_only: # Directories do not get pulled into cache return cache_path else: @@ -495,10 +495,10 @@ class AzureBlobObjectStore(ConcreteObjectStore): # return cache_path raise ObjectNotFound('objectstore.get_filename, no cache_path: %s, kwargs: %s' % (str(obj), str(kwargs))) - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): if create is True: - self.create(obj, **kwargs) - elif self.exists(obj, **kwargs): + self._create(obj, **kwargs) + elif self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Chose whether to use the dataset file itself or an alternate file if file_name: @@ -520,8 +520,8 @@ class AzureBlobObjectStore(ConcreteObjectStore): else: raise ObjectNotFound('objectstore.update_from_file, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def get_object_url(self, obj, **kwargs): - if self.exists(obj, **kwargs): + def _get_object_url(self, obj, **kwargs): + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) try: url = self.service.make_blob_url(container_name=self.container_name, blob_name=rel_path) @@ -530,7 +530,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): log.exception("Trouble generating URL for dataset '%s'", rel_path) return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return 0.0 ################## diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 49d1fa778e7..b94c9eba9ea 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -510,7 +510,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): os.path.getsize(self._get_cache_path(rel_path)), self._get_size_in_cloud(rel_path)) return False - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): in_cache = False rel_path = self._construct_path(obj, **kwargs) @@ -543,8 +543,8 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): else: return False - def create(self, obj, **kwargs): - if not self.exists(obj, **kwargs): + def _create(self, obj, **kwargs): + if not self._exists(obj, **kwargs): # Pull out locally used fields extra_dir = kwargs.get('extra_dir', None) @@ -572,26 +572,26 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_os(rel_path, from_string='') - def empty(self, obj, **kwargs): - if self.exists(obj, **kwargs): - return bool(self.size(obj, **kwargs) > 0) + def _empty(self, obj, **kwargs): + if self._exists(obj, **kwargs): + return bool(self._size(obj, **kwargs) > 0) else: raise ObjectNotFound('objectstore.empty, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) if self._in_cache(rel_path): try: return os.path.getsize(self._get_cache_path(rel_path)) except OSError as ex: log.info("Could not get size of file '%s' in local cache, will try cloud. Error: %s", rel_path, ex) - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): return self._get_size_in_cloud(rel_path) log.warning("Did not find dataset '%s', returning 0 for size", rel_path) return 0 - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): rel_path = self._construct_path(obj, **kwargs) extra_dir = kwargs.get('extra_dir', None) base_dir = kwargs.get('base_dir', None) @@ -626,10 +626,10 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): except Exception: log.exception("Could not delete key '%s' from cloud", rel_path) except OSError: - log.exception('%s delete error', self.get_filename(obj, **kwargs)) + log.exception('%s delete error', self._get_filename(obj, **kwargs)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Check cache first and get file if not there if not self._in_cache(rel_path): @@ -641,7 +641,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): base_dir = kwargs.get('base_dir', None) dir_only = kwargs.get('dir_only', False) obj_dir = kwargs.get('obj_dir', False) @@ -664,7 +664,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): if self._in_cache(rel_path): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): if dir_only: # Directories do not get pulled into cache return cache_path else: @@ -678,10 +678,10 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): % (str(obj), str(kwargs))) # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): if create: - self.create(obj, **kwargs) - if self.exists(obj, **kwargs): + self._create(obj, **kwargs) + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Chose whether to use the dataset file itself or an alternate file if file_name: @@ -703,8 +703,8 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): raise ObjectNotFound('objectstore.update_from_file, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def get_object_url(self, obj, **kwargs): - if self.exists(obj, **kwargs): + def _get_object_url(self, obj, **kwargs): + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) try: key = self.bucket.objects.get(rel_path) @@ -713,5 +713,5 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): log.exception("Trouble generating URL for dataset '%s'", rel_path) return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return 0.0 diff --git a/lib/galaxy/objectstore/pithos.py b/lib/galaxy/objectstore/pithos.py index 2d658f0133b..63bc8e23e94 100644 --- a/lib/galaxy/objectstore/pithos.py +++ b/lib/galaxy/objectstore/pithos.py @@ -221,7 +221,7 @@ class PithosObjectStore(ConcreteObjectStore): # No need to overwrite "shutdown" - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): """Check if file exists, fix if file in cache and not on Pithos+ :returns: weather the file exists remotely or in cache """ @@ -253,9 +253,9 @@ class PithosObjectStore(ConcreteObjectStore): return True return False - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """Touch a file (aka create empty), if it doesn't exist""" - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): # Pull out locally used fields extra_dir = kwargs.get('extra_dir', None) extra_dir_at_root = kwargs.get('extra_dir_at_root', False) @@ -288,18 +288,18 @@ class PithosObjectStore(ConcreteObjectStore): open(new_file, 'w').close() self.pithos.upload_from_string(rel_path, '') - def empty(self, obj, **kwargs): + def _empty(self, obj, **kwargs): """ :returns: weather the object has content :raises ObjectNotFound: """ - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): raise ObjectNotFound( 'objectstore.empty, object does not exist: {obj}, ' 'kwargs: {kwargs}'.format(obj=obj, kwargs=kwargs)) - return bool(self.size(obj, **kwargs)) + return bool(self._size(obj, **kwargs)) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): """ :returns: The size of the object, or 0 if it doesn't exist (sorry for that, not our fault, the ObjectStore interface is like that some @@ -321,7 +321,7 @@ class PithosObjectStore(ConcreteObjectStore): return 0 return int(file['content-length']) - def delete(self, obj, **kwargs): + def _delete(self, obj, **kwargs): """Delete the object :returns: weather the object was deleted """ @@ -347,13 +347,13 @@ class PithosObjectStore(ConcreteObjectStore): self.pithos.del_object(path) except OSError: log.exception( - '{0} delete error'.format(self.get_filename(obj, **kwargs))) + '{0} delete error'.format(self._get_filename(obj, **kwargs))) except ClientError as ce: log.exception('Could not delete {path} from Pithos, {err}'.format( path=path, err=ce)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): """Fetch (e.g., download) data :param start: Chunk of data starts here :param count: Fetch at most as many data, fetch all if negative @@ -369,7 +369,7 @@ class PithosObjectStore(ConcreteObjectStore): data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): """Get the expected filename with absolute path""" base_dir = kwargs.get('base_dir', None) dir_only = kwargs.get('dir_only', False) @@ -386,7 +386,7 @@ class PithosObjectStore(ConcreteObjectStore): return cache_path if self._in_cache(path): return cache_path - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): if not dir_only: self._pull_into_cache(path) return cache_path @@ -394,11 +394,11 @@ class PithosObjectStore(ConcreteObjectStore): 'objectstore.get_filename, no cache_path: {obj}, ' 'kwargs: {kwargs}'.format(obj, kwargs)) - def update_from_file(self, obj, **kwargs): + def _update_from_file(self, obj, **kwargs): """Update the store when a file is updated""" if kwargs.get('create'): - self.create(obj, **kwargs) - if not self.exists(obj, **kwargs): + self._create(obj, **kwargs) + if not self._exists(obj, **kwargs): raise ObjectNotFound( 'objectstore.update_from_file, object does not exist: {obj}, ' 'kwargs: {kwargs}'.format(obj, kwargs)) @@ -420,11 +420,11 @@ class PithosObjectStore(ConcreteObjectStore): with open(cache_path) as f: self.pithos.upload_object(obj, f) - def get_object_url(self, obj, **kwargs): + def _get_object_url(self, obj, **kwargs): """ :returns: URL for direct access, None if no object """ - if self.exists(obj, **kwargs): + if self._exists(obj, **kwargs): path = self._construct_path(obj, **kwargs) try: return self.pithos.publish_object(path) @@ -434,7 +434,7 @@ class PithosObjectStore(ConcreteObjectStore): log.exception('Kamaki: {0}'.format(ce)) return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): """ :returns: percentage indicating how full the store is """ diff --git a/lib/galaxy/objectstore/pulsar.py b/lib/galaxy/objectstore/pulsar.py index d7d1e8b7cff..c2217311ae4 100644 --- a/lib/galaxy/objectstore/pulsar.py +++ b/lib/galaxy/objectstore/pulsar.py @@ -1,6 +1,6 @@ from __future__ import absolute_import # Need to import pulsar_client absolutely. -from ..objectstore import ObjectStore +from ..objectstore import BaseObjectStore try: from pulsar.client.manager import ObjectStoreClientManager @@ -8,7 +8,7 @@ except ImportError: ObjectStoreClientManager = None -class PulsarObjectStore(ObjectStore): +class PulsarObjectStore(BaseObjectStore): """ Object store implementation that delegates to a remote Pulsar server. @@ -26,38 +26,38 @@ class PulsarObjectStore(ObjectStore): def __init__(self, config, config_xml): self.pulsar_client = self.__build_pulsar_client(config_xml) - def exists(self, obj, **kwds): + def _exists(self, obj, **kwds): return self.pulsar_client.exists(**self.__build_kwds(obj, **kwds)) def file_ready(self, obj, **kwds): return self.pulsar_client.file_ready(**self.__build_kwds(obj, **kwds)) - def create(self, obj, **kwds): + def _create(self, obj, **kwds): return self.pulsar_client.create(**self.__build_kwds(obj, **kwds)) - def empty(self, obj, **kwds): + def _empty(self, obj, **kwds): return self.pulsar_client.empty(**self.__build_kwds(obj, **kwds)) - def size(self, obj, **kwds): + def _size(self, obj, **kwds): return self.pulsar_client.size(**self.__build_kwds(obj, **kwds)) - def delete(self, obj, **kwds): + def _delete(self, obj, **kwds): return self.pulsar_client.delete(**self.__build_kwds(obj, **kwds)) # TODO: Optimize get_data. - def get_data(self, obj, **kwds): + def _get_data(self, obj, **kwds): return self.pulsar_client.get_data(**self.__build_kwds(obj, **kwds)) - def get_filename(self, obj, **kwds): + def _get_filename(self, obj, **kwds): return self.pulsar_client.get_filename(**self.__build_kwds(obj, **kwds)) - def update_from_file(self, obj, **kwds): + def _update_from_file(self, obj, **kwds): return self.pulsar_client.update_from_file(**self.__build_kwds(obj, **kwds)) - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return self.pulsar_client.get_store_usage_percent() - def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None): + def _get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None): return None def __build_kwds(self, obj, **kwds): diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 65114037c16..f01c87720c4 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -148,15 +148,15 @@ class IRODSObjectStore(DiskObjectStore): assert status == 0, '__mkcolls(): Failed to create collection: %s' % collname @local_extra_dirs - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): doi = irods.dataObjInp_t() doi.objPath = self.__get_rods_path(obj, **kwargs) log.debug('exists(): checking: %s', doi.objPath) return irods.rcObjStat(self.rods_conn, doi) is not None @local_extra_dirs - def create(self, obj, **kwargs): - if not self.exists(obj, **kwargs): + def _create(self, obj, **kwargs): + if not self._exists(obj, **kwargs): rods_path = self.__get_rods_path(obj, **kwargs) log.debug('create(): %s', rods_path) dir_only = kwargs.get('dir_only', False) @@ -177,7 +177,7 @@ class IRODSObjectStore(DiskObjectStore): assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % (rods_path, status, irods.strerror(status)) @local_extra_dirs - def empty(self, obj, **kwargs): + def _empty(self, obj, **kwargs): assert 'dir_only' not in kwargs, 'empty(): `dir_only` parameter is invalid here' h = self.__get_rods_handle(obj, **kwargs) try: @@ -186,7 +186,7 @@ class IRODSObjectStore(DiskObjectStore): # h is None raise ObjectNotFound() - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): assert 'dir_only' not in kwargs, 'size(): `dir_only` parameter is invalid here' h = self.__get_rods_handle(obj, **kwargs) try: @@ -196,7 +196,7 @@ class IRODSObjectStore(DiskObjectStore): return 0 @local_extra_dirs - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): assert 'dir_only' not in kwargs, 'delete(): `dir_only` parameter is invalid here' rods_path = self.__get_rods_path(obj, **kwargs) # __get_rods_path prepends self.root_collection_path but we are going @@ -223,7 +223,7 @@ class IRODSObjectStore(DiskObjectStore): return False @local_extra_dirs - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): log.debug('get_data(): %s') h = self.__get_rods_handle(obj, **kwargs) try: @@ -238,11 +238,11 @@ class IRODSObjectStore(DiskObjectStore): # reads data into a var, closes, and returns the var @local_extra_dirs - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): log.debug("get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.", obj.__class__.__name__, obj.id) cached_path = self.__get_cache_path(obj, **kwargs) - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): raise ObjectNotFound() # TODO: implement or define whether dir_only is valid @@ -286,11 +286,11 @@ class IRODSObjectStore(DiskObjectStore): return os.path.abspath(cached_path) @local_extra_dirs - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): assert 'dir_only' not in kwargs, 'update_from_file(): `dir_only` parameter is invalid here' # do not create if not requested - if create and not self.exists(obj, **kwargs): + if create and not self._exists(obj, **kwargs): raise ObjectNotFound() if file_name is None: @@ -311,10 +311,10 @@ class IRODSObjectStore(DiskObjectStore): status = irods.rcDataObjPut(self.rods_conn, doi, file_name) assert status == 0, 'update_from_file(): iput %s failed (%s): %s' % (doi.objPath, status, irods.strerror(status)) - def get_object_url(self, obj, **kwargs): + def _get_object_url(self, obj, **kwargs): return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return 0.0 diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index a1729498391..ca0b35a178f 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -504,7 +504,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): os.path.getsize(self._get_cache_path(rel_path)), self._get_size_in_s3(rel_path)) return False - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): in_cache = in_s3 = False rel_path = self._construct_path(obj, **kwargs) @@ -537,8 +537,8 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): else: return False - def create(self, obj, **kwargs): - if not self.exists(obj, **kwargs): + def _create(self, obj, **kwargs): + if not self._exists(obj, **kwargs): # Pull out locally used fields extra_dir = kwargs.get('extra_dir', None) @@ -572,26 +572,26 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_os(rel_path, from_string='') - def empty(self, obj, **kwargs): - if self.exists(obj, **kwargs): - return bool(self.size(obj, **kwargs) > 0) + def _empty(self, obj, **kwargs): + if self._exists(obj, **kwargs): + return bool(self._size(obj, **kwargs) > 0) else: raise ObjectNotFound('objectstore.empty, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) if self._in_cache(rel_path): try: return os.path.getsize(self._get_cache_path(rel_path)) except OSError as ex: log.info("Could not get size of file '%s' in local cache, will try S3. Error: %s", rel_path, ex) - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): return self._get_size_in_s3(rel_path) log.warning("Did not find dataset '%s', returning 0 for size", rel_path) return 0 - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): rel_path = self._construct_path(obj, **kwargs) extra_dir = kwargs.get('extra_dir', None) base_dir = kwargs.get('base_dir', None) @@ -626,10 +626,10 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): except S3ResponseError: log.exception("Could not delete key '%s' from S3", rel_path) except OSError: - log.exception('%s delete error', self.get_filename(obj, **kwargs)) + log.exception('%s delete error', self._get_filename(obj, **kwargs)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Check cache first and get file if not there if not self._in_cache(rel_path): @@ -641,7 +641,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): base_dir = kwargs.get('base_dir', None) dir_only = kwargs.get('dir_only', False) obj_dir = kwargs.get('obj_dir', False) @@ -664,7 +664,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): if self._in_cache(rel_path): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): if dir_only: # Directories do not get pulled into cache return cache_path else: @@ -678,10 +678,10 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): % (str(obj), str(kwargs))) # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): if create: - self.create(obj, **kwargs) - if self.exists(obj, **kwargs): + self._create(obj, **kwargs) + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Chose whether to use the dataset file itself or an alternate file if file_name: @@ -703,8 +703,8 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): raise ObjectNotFound('objectstore.update_from_file, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def get_object_url(self, obj, **kwargs): - if self.exists(obj, **kwargs): + def _get_object_url(self, obj, **kwargs): + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) try: key = Key(self._bucket, rel_path) @@ -713,7 +713,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): log.exception("Trouble generating URL for dataset '%s'", rel_path) return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return 0.0 def shutdown(self): From 4aa223e0a30265a8c7a2da78a70048dc9aa4867f Mon Sep 17 00:00:00 2001 From: "jalili.vahid@gmail.com" Date: Mon, 23 Mar 2020 19:14:23 -0700 Subject: [PATCH 144/324] Ensure interface methods implemented by BaseObjectStore cannot be overridden. --- lib/galaxy/objectstore/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index f826a91c701..3f200288d64 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -183,7 +183,18 @@ class IObjectStore(object): raise NotImplementedError() +class NonOverridable(type): + def __new__(mcs, name, bases, dct): + # The following is a list of methods implemented in BaseObjectStore type, which shall not be overridden. + for method in ["exists", "create", "empty", "size", "delete", "get_data", "get_filename", + "update_from_file", "get_object_url", "get_store_usage_percent", "get_store_by"]: + if bases and method in dct: + raise SyntaxError("Overriding {0} is not allowed".format(method)) + return type.__new__(mcs, name, bases, dct) + + class BaseObjectStore(IObjectStore): + __metaclass__ = NonOverridable def __init__(self, config, config_dict=None, **kwargs): """ From 01548f714444ed66ccc843d83c78b635fbb681b8 Mon Sep 17 00:00:00 2001 From: "jalili.vahid@gmail.com" Date: Mon, 23 Mar 2020 19:40:04 -0700 Subject: [PATCH 145/324] Add abstractmethod attribute to IObjectStore methods. --- lib/galaxy/objectstore/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 3f200288d64..b1e1ae4545a 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -5,6 +5,7 @@ all providers ensure that data can be accessed on the filesystem for running tools """ +import abc import logging import os import random @@ -39,6 +40,7 @@ log = logging.getLogger(__name__) class IObjectStore(object): + __metaclass__ = abc.ABCMeta """ObjectStore interface. @@ -82,10 +84,12 @@ class IObjectStore(object): 000/obj.id) """ + @abc.abstractmethod def exists(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None): """Return True if the object identified by `obj` exists, False otherwise.""" raise NotImplementedError() + @abc.abstractmethod def create(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Mark the object (`obj`) as existing in the store, but with no content. @@ -95,6 +99,7 @@ class IObjectStore(object): """ raise NotImplementedError() + @abc.abstractmethod def empty(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Test if the object identified by `obj` has content. @@ -103,6 +108,7 @@ class IObjectStore(object): """ raise NotImplementedError() + @abc.abstractmethod def size(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Return size of the object identified by `obj`. @@ -111,6 +117,7 @@ class IObjectStore(object): """ raise NotImplementedError() + @abc.abstractmethod def delete(self, obj, entire_dir=False, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Delete the object identified by `obj`. @@ -123,6 +130,7 @@ class IObjectStore(object): """ raise NotImplementedError() + @abc.abstractmethod def get_data(self, obj, start=0, count=-1, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Fetch `count` bytes of data offset by `start` bytes using `obj.id`. @@ -137,6 +145,7 @@ class IObjectStore(object): """ raise NotImplementedError() + @abc.abstractmethod def get_filename(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Get the expected filename with absolute path for object with id `obj.id`. @@ -145,6 +154,7 @@ class IObjectStore(object): """ raise NotImplementedError() + @abc.abstractmethod def update_from_file(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False, file_name=None, create=False): """ Inform the store that the file associated with `obj.id` has been updated. @@ -163,6 +173,7 @@ class IObjectStore(object): """ raise NotImplementedError() + @abc.abstractmethod def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Return the URL for direct acces if supported, otherwise return None. @@ -171,10 +182,12 @@ class IObjectStore(object): """ raise NotImplementedError() + @abc.abstractmethod def get_store_usage_percent(self): """Return the percentage indicating how full the store is.""" raise NotImplementedError() + @abc.abstractmethod def get_store_by(self, obj): """Return how object is stored (by 'uuid', 'id', or None if not yet saved). From 9dc85a604d6c7809df271332fbe1726df6027483 Mon Sep 17 00:00:00 2001 From: "jalili.vahid@gmail.com" Date: Mon, 23 Mar 2020 22:00:39 -0700 Subject: [PATCH 146/324] Make obj optional. --- lib/galaxy/objectstore/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index b1e1ae4545a..52267e9397d 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -278,8 +278,8 @@ class BaseObjectStore(IObjectStore): # job working directories. return obj.id - def _invoke(self, delegate, obj, **kwargs): - return self.__getattribute__("_" + delegate)(obj, **kwargs) + def _invoke(self, delegate, obj=None, **kwargs): + return self.__getattribute__("_" + delegate)(obj=obj, **kwargs) def exists(self, obj, **kwargs): return self._invoke('exists', obj, **kwargs) @@ -308,8 +308,8 @@ class BaseObjectStore(IObjectStore): def get_object_url(self, obj, **kwargs): return self._invoke('get_object_url', obj, **kwargs) - def get_store_usage_percent(self, **kwargs): - return self._invoke('get_store_usage_percent', **kwargs) + def get_store_usage_percent(self): + return self._invoke('get_store_usage_percent') def get_store_by(self, obj, **kwargs): return self._invoke('get_store_by', obj, **kwargs) @@ -605,7 +605,7 @@ class DiskObjectStore(ConcreteObjectStore): """ return None - def _get_store_usage_percent(self): + def _get_store_usage_percent(self, **kwargs): """Override `ObjectStore`'s stub by return percent storage used.""" st = os.statvfs(self.file_path) return (float(st.f_blocks - st.f_bavail) / st.f_blocks) * 100 From 05202d52f72144cd01f78d71c28a3e5f698cda07 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Tue, 24 Mar 2020 09:38:09 -0700 Subject: [PATCH 147/324] adding higlass IT from eu --- tools/interactive/interactivetool_higlass.xml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tools/interactive/interactivetool_higlass.xml diff --git a/tools/interactive/interactivetool_higlass.xml b/tools/interactive/interactivetool_higlass.xml new file mode 100644 index 00000000000..6d0510925da --- /dev/null +++ b/tools/interactive/interactivetool_higlass.xml @@ -0,0 +1,39 @@ + + an interactive Hi-C data visualizer + + quay.io/bgruening/galaxy-higlass + + + + 80 + + + + + + + + + + + + + + Interactive tool for visualising Hi-C data, works only for multi-cooler files which store multiple resolutions. + For a detailed documentaition please visit https://docs.higlass.io/. + + + + 10.1186/s13059-018-1486-1 + + + + From 2641c692e2ed6b19dbb45a53258a3dc327071407 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Tue, 24 Mar 2020 10:09:00 -0700 Subject: [PATCH 148/324] migrate ITs from EU accidental files more files migrate ITs from EU accidental files more files migrate ITs from EU accidental files more files migrate ITs from EU accidental files more files --- default_notebook.ipynb | 53 -------- interactivetool_askomics.xml | 70 ----------- interactivetool_bam_iobio.xml | 47 -------- interactivetool_cellxgene.xml | 114 ------------------ interactivetool_ethercalc.xml | 63 ---------- interactivetool_higlass.xml | 39 ------ interactivetool_jupyter_notebook.xml | 94 --------------- interactivetool_neo4j.xml | 41 ------- interactivetool_phinch.xml | 39 ------ interactivetool_rstudio.xml | 74 ------------ interactivetool_wallace.xml | 67 ---------- interactivetool_wilson.xml | 56 --------- .../interactivetool_climate_notebook.xml | 0 .../interactivetool_geoexplorer.xml | 0 .../interactivetool_guacamole_desktop.xml | 0 .../interactivetool_openrefine.xml | 0 .../interactive/interactivetool_panoply.xml | 0 .../interactive/interactivetool_paraview.xml | 0 .../interactive/interactivetool_pyiron.xml | 0 .../interactive/interactivetool_radiant.xml | 0 .../interactive/interactivetool_vcf_iobio.xml | 0 21 files changed, 757 deletions(-) delete mode 100644 default_notebook.ipynb delete mode 100644 interactivetool_askomics.xml delete mode 100644 interactivetool_bam_iobio.xml delete mode 100644 interactivetool_cellxgene.xml delete mode 100644 interactivetool_ethercalc.xml delete mode 100644 interactivetool_higlass.xml delete mode 100644 interactivetool_jupyter_notebook.xml delete mode 100644 interactivetool_neo4j.xml delete mode 100644 interactivetool_phinch.xml delete mode 100644 interactivetool_rstudio.xml delete mode 100644 interactivetool_wallace.xml delete mode 100644 interactivetool_wilson.xml rename interactivetool_climate_notebook.xml => tools/interactive/interactivetool_climate_notebook.xml (100%) rename interactivetool_geoexplorer.xml => tools/interactive/interactivetool_geoexplorer.xml (100%) rename interactivetool_guacamole_desktop.xml => tools/interactive/interactivetool_guacamole_desktop.xml (100%) rename interactivetool_openrefine.xml => tools/interactive/interactivetool_openrefine.xml (100%) rename interactivetool_panoply.xml => tools/interactive/interactivetool_panoply.xml (100%) rename interactivetool_paraview.xml => tools/interactive/interactivetool_paraview.xml (100%) rename interactivetool_pyiron.xml => tools/interactive/interactivetool_pyiron.xml (100%) rename interactivetool_radiant.xml => tools/interactive/interactivetool_radiant.xml (100%) rename interactivetool_vcf_iobio.xml => tools/interactive/interactivetool_vcf_iobio.xml (100%) diff --git a/default_notebook.ipynb b/default_notebook.ipynb deleted file mode 100644 index e9573752478..00000000000 --- a/default_notebook.ipynb +++ /dev/null @@ -1,53 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Welcome to the interactive Galaxy IPython Notebook." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can access your data via the dataset number. Using a Python kernel, you can access dataset number 42 with ``handle = open(get(42), 'r')``.\n", - "To save data, write your data to a file, and then call ``put('filename.txt')``. The dataset will then be available in your galaxy history.\n
", - "When using a non-Python kernel, ``get`` and ``put`` are available as command-line tools, which can be accessed using system calls in R, Julia, and Ruby. For example, to read dataset number 42 into R, you can write ```handle <- file(system('get -i 42', intern = TRUE))```.\n", - "To save data in R, write the data to a file and then call ``system('put -p filename.txt')``.\n", - "Notebooks can be saved to Galaxy by clicking the large green button at the top right of the IPython interface.
\n", - "More help and informations can be found on the project [website](https://github.com/bgruening/docker-jupyter-notebook)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.10" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/interactivetool_askomics.xml b/interactivetool_askomics.xml deleted file mode 100644 index 94cf111a2f3..00000000000 --- a/interactivetool_askomics.xml +++ /dev/null @@ -1,70 +0,0 @@ - - a visual SPARQL query builder - - askomics/flaskomics-with-dependencies:3.2.0 - - - - 5000 - /loginapikey/${__user_name__} - - - - - true - ${__user_name__} - Galaxy - ${__user_name__} - ${__user_email__} - ${__user_name__} - - $__galaxy_url__ - - - prod - 1 - Galaxy - AskOmics Interactive Tool for Galaxy - /tmp/askomics-it - /tmp/askomics-it/database.db - http://localhost:5000 - - true - - 85000 - 65000 - - - - - - - - - - - - - AskOmics is a visual SPARQL query interface supporting both intuitive data integration and - querying while shielding the user from most of the technical difficulties underlying RDF and SPARQL. - - diff --git a/interactivetool_bam_iobio.xml b/interactivetool_bam_iobio.xml deleted file mode 100644 index 6c78eaa335b..00000000000 --- a/interactivetool_bam_iobio.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - qiaoy/iobio-bundle.bam-iobio:1.0-ondemand - - - - 80 - - - - /tmp/app.conf && - mv /tmp/app.conf /etc/supervisor.d/app.conf && - - /usr/bin/supervisord -c /etc/supervisord.conf - ]]> - - - - - - - - - - - BAM iobio visualisation. - - diff --git a/interactivetool_cellxgene.xml b/interactivetool_cellxgene.xml deleted file mode 100644 index 3c99f3a393b..00000000000 --- a/interactivetool_cellxgene.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - quay.io/galaxy/cellxgene-galaxy-ie:ie2 - - - - 80 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - experimental_annotations['experimental_annotations_select'] == 'enable' - - - - - - An interactive explorer for single-cell transcriptomics (AnnData formatted) data - - cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the Human Cell Atlas. Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data. - - Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data. - https://github.com/chanzuckerberg/cellxgene - - - 10.5281/zenodo.3554576 - - diff --git a/interactivetool_ethercalc.xml b/interactivetool_ethercalc.xml deleted file mode 100644 index 88676c1e997..00000000000 --- a/interactivetool_ethercalc.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - shiltemann/ethercalc-galaxy-ie:17.05 - - - - 8000 - - - loading.txt - && - curl --include --request PUT --header "Content-Type: text/csv" --data-binary @loading.txt http://localhost:8000/_/galaxy - && - - ## remove dump file so this doesnt appear in audit trail - rm /dump.json - && - - ## load dataset into worksheet - curl --include --request PUT --header "Content-Type: text/csv" --data-binary @$infile http://localhost:8000/_/galaxy - && - - tail -f /etc/hosts - - ]]> - - - -&1) -while [[ \${STATUS} =~ "refused" ]] -do - echo "waiting for ethercalc: \$STATUS \n" - STATUS=\$(curl --include 'http://localhost:8000/_/galaxy' 2>&1) - sleep 2 -done - ]]> - - - - - - - - - - - - EtherCalc is a web spreadsheet. - https://ethercalc.net - - diff --git a/interactivetool_higlass.xml b/interactivetool_higlass.xml deleted file mode 100644 index 6d0510925da..00000000000 --- a/interactivetool_higlass.xml +++ /dev/null @@ -1,39 +0,0 @@ - - an interactive Hi-C data visualizer - - quay.io/bgruening/galaxy-higlass - - - - 80 - - - - - - - - - - - - - - Interactive tool for visualising Hi-C data, works only for multi-cooler files which store multiple resolutions. - For a detailed documentaition please visit https://docs.higlass.io/. - - - - 10.1186/s13059-018-1486-1 - - - - diff --git a/interactivetool_jupyter_notebook.xml b/interactivetool_jupyter_notebook.xml deleted file mode 100644 index c86717460df..00000000000 --- a/interactivetool_jupyter_notebook.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - quay.io/bgruening/docker-jupyter-notebook:ie2 - - - - 8888 - ipython/lab - - - - $__history_id__ - $__galaxy_url__ - 8080 - $__galaxy_url__ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations, - visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, - machine learning, and much more. - - Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to - do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. - - You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. - If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. - - You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. - - diff --git a/interactivetool_neo4j.xml b/interactivetool_neo4j.xml deleted file mode 100644 index 0b31a8b303e..00000000000 --- a/interactivetool_neo4j.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - quay.io/sanbi-sa/neo_ie:3.1.9 - - - - 80 - - - - - 2345 - 2345 - false - - - - - - - - - - - - - Neo4j is a highly scalable, robust native graph database. - - diff --git a/interactivetool_phinch.xml b/interactivetool_phinch.xml deleted file mode 100644 index 382cf32f0a9..00000000000 --- a/interactivetool_phinch.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - shiltemann/docker-phinch-galaxy:16.04 - - - - 80 - - - &1 > /var/log/phinch.log - - ]]> - - - - - - - - - - - Interactive tool for visualising Biom data. - - diff --git a/interactivetool_rstudio.xml b/interactivetool_rstudio.xml deleted file mode 100644 index 63e0eb3adb0..00000000000 --- a/interactivetool_rstudio.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - quay.io/erasche/docker-rstudio-notebook:19.09 - - - - 80 - rstudio/ - - - - ${__app__.security.encode_id($jupyter_notebook.history_id)} - ${__app__.config.galaxy_infrastructure_url} - 8080 - ${__app__.config.galaxy_infrastructure_url} - true - true - - #if $__user__: - #for $api_key in $__user__.api_keys: - ${api_key.key} - #break - #end for - #end if - - - - - - - - - - - - - - - - - - - - This familiar R analysis software suite will let you explore your - datasets in depth. Comes with ggplot2, RODBC, maps, shinyapps, knitr, - LaTeX, bioconductor, cummeRbund, and many more pre-installed packages. - - Galaxy offers you to use RStudio directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to - do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in RStudio. - - The convenience functions gx_put() and gx_get() are available to you to interact with your current Galaxy history. You can save your workspace with gx_save(). - - For example, gx_get(42) will fetch dataset 42 from your history and return the file location - - diff --git a/interactivetool_wallace.xml b/interactivetool_wallace.xml deleted file mode 100644 index 4d8e2c5353b..00000000000 --- a/interactivetool_wallace.xml +++ /dev/null @@ -1,67 +0,0 @@ - - Webbased Interactive modeling of species niches and distributions - - ylebras/wallace-docker - - - - 3838 - /sample-apps/SIG/wallace/shiny/ - - - - - ${__app__.security.encode_id($outfile.history_id)} - ${__app__.config.galaxy_infrastructure_url} - 8080 - ${__app__.config.galaxy_infrastructure_url} - - #if $__user__: - #for $api_key in $__user__.api_keys: - ${api_key.key} - #break - #end for - #end if - - - - - - - - - - - - - -`_ is a flexible platform for reproducible modeling of species niches and distributions. - -.. class:: infomark - -Example input file (TAB separated):: - - "name" "longitude" "latitude" "countryCode" - Accipiter striatus Vieillot, 1808 -60.291838 46.328137 CA - Accipiter striatus Vieillot, 1808 -114.58927 35.022485 US - Accipiter striatus Vieillot, 1808 -93.37406 30.00586 US - Accipiter striatus Vieillot, 1808 -79.336288 43.682218 CA - Accipiter striatus Vieillot, 1808 -109.156024 31.904185 US - Accipiter striatus Vieillot, 1808 -71.098031 42.297408 US - Accipiter striatus Vieillot, 1808 -110.927215 32.18203 US - -]]> - - - 10.1111/2041-210X.12945 - - diff --git a/interactivetool_wilson.xml b/interactivetool_wilson.xml deleted file mode 100644 index 7f575c43a63..00000000000 --- a/interactivetool_wilson.xml +++ /dev/null @@ -1,56 +0,0 @@ - - Webbased Interactive Omics visualization - - loosolab/wilson:2.1.1 - - - - 3838 - - - - feature_selection - true - - /home/shiny/.Renviron && - echo "WILSON_BLACKLIST_EXAMPLES=\$WILSON_BLACKLIST_EXAMPLES" >> /home/shiny/.Renviron && - ln -s ${infile} /srv/shiny-server/external_data/input.clarion && - exec shiny-server 2>&1 - ]]> - - - - - - - - - - -`_ - -.. class:: infomark - -Wilson uses the CLARION file format, which is a generic file format for quantitative comparisons of high throughput screens. - -CLARION is a data format specially developed to be used with Wilson, which relies on a tab-delimited table with -a metadata header to describe the following columns. It is based on the Summarized Experiment format and supports -all types of data which can be reduced to features and their annotation (e.g. genes, transcripts, proteins, probes) -with assigned numerical values (e.g. count, score, log2foldchange, z-score, p-value). Most result tables derived from RNA-Seq, -ChIP/ATAC-Seq, Proteomics, Microarrays, and many other analyses can thus be easily reformatted to become compatible -without having to modify the code of Wilson for each specific experiment. - -Please check the following link for details considering the `CLARION format `_. - - - -]]> - - - 10.1093/bioinformatics/bty711 - - diff --git a/interactivetool_climate_notebook.xml b/tools/interactive/interactivetool_climate_notebook.xml similarity index 100% rename from interactivetool_climate_notebook.xml rename to tools/interactive/interactivetool_climate_notebook.xml diff --git a/interactivetool_geoexplorer.xml b/tools/interactive/interactivetool_geoexplorer.xml similarity index 100% rename from interactivetool_geoexplorer.xml rename to tools/interactive/interactivetool_geoexplorer.xml diff --git a/interactivetool_guacamole_desktop.xml b/tools/interactive/interactivetool_guacamole_desktop.xml similarity index 100% rename from interactivetool_guacamole_desktop.xml rename to tools/interactive/interactivetool_guacamole_desktop.xml diff --git a/interactivetool_openrefine.xml b/tools/interactive/interactivetool_openrefine.xml similarity index 100% rename from interactivetool_openrefine.xml rename to tools/interactive/interactivetool_openrefine.xml diff --git a/interactivetool_panoply.xml b/tools/interactive/interactivetool_panoply.xml similarity index 100% rename from interactivetool_panoply.xml rename to tools/interactive/interactivetool_panoply.xml diff --git a/interactivetool_paraview.xml b/tools/interactive/interactivetool_paraview.xml similarity index 100% rename from interactivetool_paraview.xml rename to tools/interactive/interactivetool_paraview.xml diff --git a/interactivetool_pyiron.xml b/tools/interactive/interactivetool_pyiron.xml similarity index 100% rename from interactivetool_pyiron.xml rename to tools/interactive/interactivetool_pyiron.xml diff --git a/interactivetool_radiant.xml b/tools/interactive/interactivetool_radiant.xml similarity index 100% rename from interactivetool_radiant.xml rename to tools/interactive/interactivetool_radiant.xml diff --git a/interactivetool_vcf_iobio.xml b/tools/interactive/interactivetool_vcf_iobio.xml similarity index 100% rename from interactivetool_vcf_iobio.xml rename to tools/interactive/interactivetool_vcf_iobio.xml From b2ad091f6a5c323f5d0622b4cbfab64033284cb6 Mon Sep 17 00:00:00 2001 From: "jalili.vahid@gmail.com" Date: Tue, 24 Mar 2020 03:23:26 -0700 Subject: [PATCH 149/324] Do not declare methods as non-overridable, because ... --- lib/galaxy/objectstore/__init__.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 52267e9397d..c2552ca3334 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -196,18 +196,7 @@ class IObjectStore(object): raise NotImplementedError() -class NonOverridable(type): - def __new__(mcs, name, bases, dct): - # The following is a list of methods implemented in BaseObjectStore type, which shall not be overridden. - for method in ["exists", "create", "empty", "size", "delete", "get_data", "get_filename", - "update_from_file", "get_object_url", "get_store_usage_percent", "get_store_by"]: - if bases and method in dct: - raise SyntaxError("Overriding {0} is not allowed".format(method)) - return type.__new__(mcs, name, bases, dct) - - class BaseObjectStore(IObjectStore): - __metaclass__ = NonOverridable def __init__(self, config, config_dict=None, **kwargs): """ From 9e48192ea97bac9a9960bb520d7b8d87b0799554 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 25 Mar 2020 17:20:07 +0100 Subject: [PATCH 150/324] Add right arrow --- .../components/Workflow/Editor/Node.vue | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index d3d91f17886..a5ef377e1f4 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -10,6 +10,16 @@ > + + + From 6bc8cf329ee6806af6ff5f3c5dcfd1ed9cbc18b1 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 25 Mar 2020 19:18:56 +0100 Subject: [PATCH 151/324] Fetch path --- .../components/Workflow/Editor/Node.vue | 3 + .../components/Workflow/Editor/utilities.js | 136 ++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index a5ef377e1f4..7ddaddf95eb 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -47,6 +47,7 @@ import BootstrapVue from "bootstrap-vue"; import WorkflowIcons from "components/Workflow/icons"; import LoadingSpan from "components/LoadingSpan"; import { getGalaxyInstance } from "app"; +import { getToolRecommendations } from "./utilities"; Vue.use(BootstrapVue); @@ -99,6 +100,8 @@ export default { }, onGetRecommendations() { console.log("Clicked recommendations"); + console.log(this); + getToolRecommendations(this); } }, }; diff --git a/client/galaxy/scripts/components/Workflow/Editor/utilities.js b/client/galaxy/scripts/components/Workflow/Editor/utilities.js index dcd01b05aaf..7e539525496 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/utilities.js +++ b/client/galaxy/scripts/components/Workflow/Editor/utilities.js @@ -206,3 +206,139 @@ export function saveAs(workflow) { Cancel: hide_modal, }); } + +function getToolId(toolId) { + if (toolId !== undefined && toolId !== null && toolId.indexOf("/") > -1) { + let toolIdSlash = toolId.split("/"); + toolId = toolIdSlash[toolIdSlash.length - 2]; + } + return toolId; +} + +function get_workflow_path(wf_steps, current_node_id, current_node_name) { + let steps = {}; + let step_names = {}; + for (let stp_idx in wf_steps.steps) { + let step = wf_steps.steps[stp_idx]; + let input_connections = step.input_connections; + step_names[step.id] = getToolId(step.content_id); + for (let ic_idx in input_connections) { + let ic = input_connections[ic_idx]; + if(ic !== null && ic !== undefined) { + let prev_conn = []; + for (let conn of ic) { + prev_conn.push(conn.id.toString()); + } + steps[step.id.toString()] = prev_conn; + } + } + } + // recursive call to determine path + function read_paths(node_id, ph) { + for (let st in steps) { + if (parseInt(st) === parseInt(node_id)) { + let parent_id = parseInt(steps[st][0]); + if (parent_id !== undefined && parent_id !== null) { + ph.push(parent_id); + if (steps[parent_id] !== undefined && steps[parent_id] !== null) { + read_paths(parent_id, ph); + } + } + } + } + return ph; + } + let ph = []; + let step_names_list = []; + ph.push(current_node_id); + ph = read_paths(current_node_id, ph); + for (let s_idx of ph) { + let s_name = step_names[s_idx.toString()]; + if (s_name !== undefined && s_name !== null) { + step_names_list.push(s_name); + } + } + return step_names_list.join(","); +} + +export function getToolRecommendations(propsData) { + let workflow_simple = propsData.node.app.to_simple(), //window.workflow_globals.workflow.to_simple(), + node = propsData.node, + toolId = getToolId(node.content_id); + console.log(workflow_simple); + console.log(node); + let tool_sequence = get_workflow_path(workflow_simple, node.id, toolId); + console.log(tool_sequence); + // remove ui-modal if present + /*let $modal = $(".modal-tool-recommendation"); + if ($modal.length > 0) { + $modal.remove(); + } + // create new modal + let modal = new Modal.View({ + title: "Recommended tools", + body: "
Loading tools ...
", + height: "230", + width: "250", + closing_events: true, + title_separator: true + }); + modal.$el.addClass("modal-tool-recommendation"); + modal.$el.find(".modal-header").attr("title", "The recommended tools are shown in the decreasing order of their scores predicted using machine learning analysis on workflows. A tool with a higher score (closer to 100%) may fit better as the following tool than a tool with a lower score. Please click on one of the following/recommended tools to have it on the workflow editor."); + modal.$el.find(".modal-body").css("overflow", "auto"); + modal.show(); + // fetch recommended tools + Utils.request({ + type: "POST", + url: `${getAppRoot()}api/workflows/get_tool_predictions`, + data: {"tool_sequence": tool_sequence}, + success: function(data) { + let predTemplate = "
"; + let predictedData = data.predicted_data; + let outputDatatypes = predictedData["o_extensions"]; + let predictedDataChildren = predictedData.children; + let noRecommendationsMessage = "No tool recommendations"; + if (predictedDataChildren.length > 0) { + let compatibleTools = {}; + // filter results based on datatype compatibility + for (const [index, name_obj] of predictedDataChildren.entries()) { + let inputDatatypes = name_obj["i_extensions"]; + for (const out_t of outputDatatypes.entries()) { + for(const in_t of inputDatatypes.entries()) { + if ((window.workflow_globals.app.isSubType(out_t[1], in_t[1]) === true) || + out_t[1] === "input" || + out_t[1] === "_sniff_" || + out_t[1] === "input_collection") { + compatibleTools[name_obj["tool_id"]] = name_obj["name"]; + break + } + } + } + } + predTemplate += "
"; + if (Object.keys(compatibleTools).length > 0 && predictedData["is_deprecated"] === false) { + for (let id in compatibleTools) { + predTemplate += "" + compatibleTools[id]; + predTemplate += "
"; + } + } + else if (predictedData["is_deprecated"] === true) { + predTemplate += predictedData["message"]; + } + else { + predTemplate += noRecommendationsMessage; + } + predTemplate += "
"; + } + else { + predTemplate += noRecommendationsMessage; + } + predTemplate += "
"; + modal.$body.html(predTemplate); + $(".pred-tool").click(e => { + workflow_globals.app.add_node_for_tool(e.target.id, e.target.id); + modal.hide(); + }); + } + });*/ + } From f94365ef931ced126fdd5217e9da24162c566d40 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 25 Mar 2020 19:59:46 +0100 Subject: [PATCH 152/324] Add modal --- .../scripts/components/Workflow/Editor/utilities.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/utilities.js b/client/galaxy/scripts/components/Workflow/Editor/utilities.js index 7e539525496..79c341000d8 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/utilities.js +++ b/client/galaxy/scripts/components/Workflow/Editor/utilities.js @@ -7,6 +7,7 @@ import WorkflowIcons from "components/Workflow/icons"; import { DefaultForm, ToolForm } from "mvc/workflow/workflow-forms"; import { loadWorkflow } from "./services"; import { hide_modal, show_message, show_modal } from "layout/modal"; +import Modal from "mvc/ui/ui-modal"; export function copyIntoWorkflow(workflow, id = null, stepCount = null) { const _copy_into_workflow_ajax = () => { @@ -262,15 +263,16 @@ function get_workflow_path(wf_steps, current_node_id, current_node_name) { } export function getToolRecommendations(propsData) { - let workflow_simple = propsData.node.app.to_simple(), //window.workflow_globals.workflow.to_simple(), + let workflow_simple = propsData.node.app.to_simple(), node = propsData.node, toolId = getToolId(node.content_id); console.log(workflow_simple); console.log(node); let tool_sequence = get_workflow_path(workflow_simple, node.id, toolId); console.log(tool_sequence); + // remove ui-modal if present - /*let $modal = $(".modal-tool-recommendation"); + let $modal = $(".modal-tool-recommendation"); if ($modal.length > 0) { $modal.remove(); } @@ -305,7 +307,7 @@ export function getToolRecommendations(propsData) { let inputDatatypes = name_obj["i_extensions"]; for (const out_t of outputDatatypes.entries()) { for(const in_t of inputDatatypes.entries()) { - if ((window.workflow_globals.app.isSubType(out_t[1], in_t[1]) === true) || + if ((propsData.node.app.isSubType(out_t[1], in_t[1]) === true) || out_t[1] === "input" || out_t[1] === "_sniff_" || out_t[1] === "input_collection") { @@ -340,5 +342,5 @@ export function getToolRecommendations(propsData) { modal.hide(); }); } - });*/ + }); } From eb42ee88426d47747d30357117cd52ea4a6c0401 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Thu, 26 Mar 2020 00:59:41 +0100 Subject: [PATCH 153/324] Add utilities and vue --- .../components/Workflow/Editor/Node.vue | 4 +- .../Editor/WorkflowRecommendationsVue.vue | 66 ++++++++ .../components/Workflow/Editor/utilities.js | 137 ---------------- .../Editor/workflowRecommendations.js | 148 ++++++++++++++++++ 4 files changed, 215 insertions(+), 140 deletions(-) create mode 100644 client/galaxy/scripts/components/Workflow/Editor/WorkflowRecommendationsVue.vue create mode 100644 client/galaxy/scripts/components/Workflow/Editor/workflowRecommendations.js diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index 7ddaddf95eb..2f5c92c28c6 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -47,7 +47,7 @@ import BootstrapVue from "bootstrap-vue"; import WorkflowIcons from "components/Workflow/icons"; import LoadingSpan from "components/LoadingSpan"; import { getGalaxyInstance } from "app"; -import { getToolRecommendations } from "./utilities"; +import { getToolRecommendations } from "./workflowRecommendations"; Vue.use(BootstrapVue); @@ -99,8 +99,6 @@ export default { this.node.clone(); }, onGetRecommendations() { - console.log("Clicked recommendations"); - console.log(this); getToolRecommendations(this); } }, diff --git a/client/galaxy/scripts/components/Workflow/Editor/WorkflowRecommendationsVue.vue b/client/galaxy/scripts/components/Workflow/Editor/WorkflowRecommendationsVue.vue new file mode 100644 index 00000000000..64b8b59690b --- /dev/null +++ b/client/galaxy/scripts/components/Workflow/Editor/WorkflowRecommendationsVue.vue @@ -0,0 +1,66 @@ + + + diff --git a/client/galaxy/scripts/components/Workflow/Editor/utilities.js b/client/galaxy/scripts/components/Workflow/Editor/utilities.js index 79c341000d8..dd705c7c5be 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/utilities.js +++ b/client/galaxy/scripts/components/Workflow/Editor/utilities.js @@ -207,140 +207,3 @@ export function saveAs(workflow) { Cancel: hide_modal, }); } - -function getToolId(toolId) { - if (toolId !== undefined && toolId !== null && toolId.indexOf("/") > -1) { - let toolIdSlash = toolId.split("/"); - toolId = toolIdSlash[toolIdSlash.length - 2]; - } - return toolId; -} - -function get_workflow_path(wf_steps, current_node_id, current_node_name) { - let steps = {}; - let step_names = {}; - for (let stp_idx in wf_steps.steps) { - let step = wf_steps.steps[stp_idx]; - let input_connections = step.input_connections; - step_names[step.id] = getToolId(step.content_id); - for (let ic_idx in input_connections) { - let ic = input_connections[ic_idx]; - if(ic !== null && ic !== undefined) { - let prev_conn = []; - for (let conn of ic) { - prev_conn.push(conn.id.toString()); - } - steps[step.id.toString()] = prev_conn; - } - } - } - // recursive call to determine path - function read_paths(node_id, ph) { - for (let st in steps) { - if (parseInt(st) === parseInt(node_id)) { - let parent_id = parseInt(steps[st][0]); - if (parent_id !== undefined && parent_id !== null) { - ph.push(parent_id); - if (steps[parent_id] !== undefined && steps[parent_id] !== null) { - read_paths(parent_id, ph); - } - } - } - } - return ph; - } - let ph = []; - let step_names_list = []; - ph.push(current_node_id); - ph = read_paths(current_node_id, ph); - for (let s_idx of ph) { - let s_name = step_names[s_idx.toString()]; - if (s_name !== undefined && s_name !== null) { - step_names_list.push(s_name); - } - } - return step_names_list.join(","); -} - -export function getToolRecommendations(propsData) { - let workflow_simple = propsData.node.app.to_simple(), - node = propsData.node, - toolId = getToolId(node.content_id); - console.log(workflow_simple); - console.log(node); - let tool_sequence = get_workflow_path(workflow_simple, node.id, toolId); - console.log(tool_sequence); - - // remove ui-modal if present - let $modal = $(".modal-tool-recommendation"); - if ($modal.length > 0) { - $modal.remove(); - } - // create new modal - let modal = new Modal.View({ - title: "Recommended tools", - body: "
Loading tools ...
", - height: "230", - width: "250", - closing_events: true, - title_separator: true - }); - modal.$el.addClass("modal-tool-recommendation"); - modal.$el.find(".modal-header").attr("title", "The recommended tools are shown in the decreasing order of their scores predicted using machine learning analysis on workflows. A tool with a higher score (closer to 100%) may fit better as the following tool than a tool with a lower score. Please click on one of the following/recommended tools to have it on the workflow editor."); - modal.$el.find(".modal-body").css("overflow", "auto"); - modal.show(); - // fetch recommended tools - Utils.request({ - type: "POST", - url: `${getAppRoot()}api/workflows/get_tool_predictions`, - data: {"tool_sequence": tool_sequence}, - success: function(data) { - let predTemplate = "
"; - let predictedData = data.predicted_data; - let outputDatatypes = predictedData["o_extensions"]; - let predictedDataChildren = predictedData.children; - let noRecommendationsMessage = "No tool recommendations"; - if (predictedDataChildren.length > 0) { - let compatibleTools = {}; - // filter results based on datatype compatibility - for (const [index, name_obj] of predictedDataChildren.entries()) { - let inputDatatypes = name_obj["i_extensions"]; - for (const out_t of outputDatatypes.entries()) { - for(const in_t of inputDatatypes.entries()) { - if ((propsData.node.app.isSubType(out_t[1], in_t[1]) === true) || - out_t[1] === "input" || - out_t[1] === "_sniff_" || - out_t[1] === "input_collection") { - compatibleTools[name_obj["tool_id"]] = name_obj["name"]; - break - } - } - } - } - predTemplate += "
"; - if (Object.keys(compatibleTools).length > 0 && predictedData["is_deprecated"] === false) { - for (let id in compatibleTools) { - predTemplate += "" + compatibleTools[id]; - predTemplate += "
"; - } - } - else if (predictedData["is_deprecated"] === true) { - predTemplate += predictedData["message"]; - } - else { - predTemplate += noRecommendationsMessage; - } - predTemplate += "
"; - } - else { - predTemplate += noRecommendationsMessage; - } - predTemplate += "
"; - modal.$body.html(predTemplate); - $(".pred-tool").click(e => { - workflow_globals.app.add_node_for_tool(e.target.id, e.target.id); - modal.hide(); - }); - } - }); - } diff --git a/client/galaxy/scripts/components/Workflow/Editor/workflowRecommendations.js b/client/galaxy/scripts/components/Workflow/Editor/workflowRecommendations.js new file mode 100644 index 00000000000..ca4f46b652c --- /dev/null +++ b/client/galaxy/scripts/components/Workflow/Editor/workflowRecommendations.js @@ -0,0 +1,148 @@ +import _ from "underscore"; +import $ from "jquery"; +import Vue from "vue"; +import WorkflowToolRecommendations from "components/Workflow/Editor/WorkflowRecommendationsVue"; + +function getToolId(toolId) { + if (toolId !== undefined && toolId !== null && toolId.indexOf("/") > -1) { + let toolIdSlash = toolId.split("/"); + toolId = toolIdSlash[toolIdSlash.length - 2]; + } + return toolId; +} + +function getWorkflowPath(wf_steps, current_node_id, current_node_name) { + let steps = {}; + let step_names = {}; + for (let stp_idx in wf_steps.steps) { + let step = wf_steps.steps[stp_idx]; + let input_connections = step.input_connections; + step_names[step.id] = getToolId(step.content_id); + for (let ic_idx in input_connections) { + let ic = input_connections[ic_idx]; + if(ic !== null && ic !== undefined) { + let prev_conn = []; + for (let conn of ic) { + prev_conn.push(conn.id.toString()); + } + steps[step.id.toString()] = prev_conn; + } + } + } + // recursive call to determine path + function readPaths(node_id, ph) { + for (let st in steps) { + if (parseInt(st) === parseInt(node_id)) { + let parent_id = parseInt(steps[st][0]); + if (parent_id !== undefined && parent_id !== null) { + ph.push(parent_id); + if (steps[parent_id] !== undefined && steps[parent_id] !== null) { + readPaths(parent_id, ph); + } + } + } + } + return ph; + } + let ph = []; + let step_names_list = []; + ph.push(current_node_id); + ph = readPaths(current_node_id, ph); + for (let s_idx of ph) { + let s_name = step_names[s_idx.toString()]; + if (s_name !== undefined && s_name !== null) { + step_names_list.push(s_name); + } + } + return step_names_list.join(","); +} + +export function getToolRecommendations(props) { + const workflowSimple = props.node.app.to_simple(); + const node = props.node; + const toolId = getToolId(node.content_id); + const toolSequence = getWorkflowPath(workflowSimple, node.id, toolId); + + // show tool recommendations + const ToolRecommendationInstance = Vue.extend(WorkflowToolRecommendations); + const vm = document.createElement("div"); + $("body").append(vm); + const instance = new ToolRecommendationInstance({ + propsData: { + workflowManager: props, + toolSequence: toolSequence + }, + }); + instance.$mount(vm); + + // remove ui-modal if present + /*let $modal = $(".modal-tool-recommendation"); + if ($modal.length > 0) { + $modal.remove(); + } + // create new modal + let modal = new Modal.View({ + title: "Recommended tools", + body: "
Loading tools ...
", + height: "230", + width: "250", + closing_events: true, + title_separator: true + }); + modal.$el.addClass("modal-tool-recommendation"); + modal.$el.find(".modal-header").attr("title", "The recommended tools are shown in the decreasing order of their scores predicted using machine learning analysis on workflows. A tool with a higher score (closer to 100%) may fit better as the following tool than a tool with a lower score. Please click on one of the following/recommended tools to have it on the workflow editor."); + modal.$el.find(".modal-body").css("overflow", "auto"); + modal.show(); + axios + .post(`${getAppRoot()}api/workflows/get_tool_predictions`, { + tool_sequence: toolSequence, + }) + .then((data) => { + let predTemplate = "
"; + let predictedData = data.predicted_data; + let outputDatatypes = predictedData["o_extensions"]; + let predictedDataChildren = predictedData.children; + let noRecommendationsMessage = "No tool recommendations"; + if (predictedDataChildren.length > 0) { + let compatibleTools = {}; + // filter results based on datatype compatibility + for (const [index, name_obj] of predictedDataChildren.entries()) { + let inputDatatypes = name_obj["i_extensions"]; + for (const out_t of outputDatatypes.entries()) { + for(const in_t of inputDatatypes.entries()) { + if ((propsData.node.app.isSubType(out_t[1], in_t[1]) === true) || + out_t[1] === "input" || + out_t[1] === "_sniff_" || + out_t[1] === "input_collection") { + compatibleTools[name_obj["tool_id"]] = name_obj["name"]; + break + } + } + } + } + predTemplate += "
"; + if (Object.keys(compatibleTools).length > 0 && predictedData["is_deprecated"] === false) { + for (let id in compatibleTools) { + predTemplate += "" + compatibleTools[id]; + predTemplate += "
"; + } + } + else if (predictedData["is_deprecated"] === true) { + predTemplate += predictedData["message"]; + } + else { + predTemplate += noRecommendationsMessage; + } + predTemplate += "
"; + } + else { + predTemplate += noRecommendationsMessage; + } + predTemplate += "
"; + modal.$body.html(predTemplate); + $(".pred-tool").click(e => { + workflow_globals.app.add_node_for_tool(e.target.id, e.target.id); + modal.hide(); + }); + });*/ + } From 8fd03451c4300cfaac310c6f0f56d69f03838934 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Thu, 26 Mar 2020 12:00:20 +0100 Subject: [PATCH 154/324] Load data via ajax --- .../Editor/WorkflowRecommendationsVue.vue | 84 +++++++++++++++---- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/WorkflowRecommendationsVue.vue b/client/galaxy/scripts/components/Workflow/Editor/WorkflowRecommendationsVue.vue index 64b8b59690b..dae2aa37128 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/WorkflowRecommendationsVue.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/WorkflowRecommendationsVue.vue @@ -1,22 +1,31 @@ @@ -116,7 +115,7 @@ export default { const clientW = svgElem.clientWidth; const translateX = parseInt(clientW * 0.15); - svgElem.setAttribute("viewBox", -translateX + " 0 " + (0.5 * clientW) + " " + clientH); + svgElem.setAttribute("viewBox", -translateX + " 0 " + 0.5 * clientW + " " + clientH); svgElem.setAttribute("preserveAspectRatio", "xMidYMid meet"); const tree = d3.layout.tree().size([clientH, clientW]); From e158231becb7b37e16829bdd73e5a74fc56d1f01 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Mon, 30 Mar 2020 00:14:53 +0200 Subject: [PATCH 177/324] Add header to popover --- client/galaxy/style/scss/base.scss | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index bec0cac5698..8c0502e238f 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1675,20 +1675,10 @@ body.reports { } } -.wf-recommendation-view { +.workflow-recommendations { display: block; - .wf-tools-dialog { - width: 20%; - } - .wf-tools-body { - overflow: auto; - height: 50%; - } - .wf-tools-background { - background: $brand-primary; - color: $white; - } - .wf-tool-content { - border: solid 1px $brand-primary; + .header-background { + border-bottom: solid 1px $brand-primary; + margin-bottom: 0.5rem; } } From 20f52f2a10d12924b8af294309acdc2911bf7c0a Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Mon, 30 Mar 2020 01:16:56 +0200 Subject: [PATCH 178/324] Add event --- client/galaxy/scripts/components/Workflow/Editor/Node.vue | 5 ++++- .../scripts/components/Workflow/Editor/Recommendations.vue | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index 5a915cb87c0..800730641be 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -113,7 +113,9 @@ export default { onClone() { this.node.clone(); }, - onCreate(toolId) { + onCreate(toolId, event) { + console.log(toolId); + console.log(event); const requestData = { tool_id: toolId, type: "tool", @@ -122,6 +124,7 @@ export default { getModule(requestData).then((response) => { var node = this.node.app.create_node("tool", response.name, toolId); this.node.app.set_node(node, response); + event.srcElement.getRootNode().remove(); }); }, }, diff --git a/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue b/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue index 7c125f73c0a..8f32d4492ae 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue @@ -1,9 +1,13 @@ \ No newline at end of file + }, +}; + diff --git a/client/galaxy/scripts/components/RuleBuilder/IdentifierDisplay.vue b/client/galaxy/scripts/components/RuleBuilder/IdentifierDisplay.vue index 9cfb211924e..54f1ce1ca4b 100644 --- a/client/galaxy/scripts/components/RuleBuilder/IdentifierDisplay.vue +++ b/client/galaxy/scripts/components/RuleBuilder/IdentifierDisplay.vue @@ -11,7 +11,7 @@ import _l from "utils/localization"; import RuleDefs from "mvc/rules/rule-definitions"; const MAPPING_TARGETS = RuleDefs.MAPPING_TARGETS; -export default{ +export default { props: { type: { type: String, @@ -49,6 +49,6 @@ export default{ columnsLabel() { return RuleDefs.columnDisplay(this.columns, this.colHeaders); }, - } -} - \ No newline at end of file + }, +}; + diff --git a/client/galaxy/scripts/components/RuleBuilder/RegularExpressionInput.vue b/client/galaxy/scripts/components/RuleBuilder/RegularExpressionInput.vue index 2cd375bc95d..36f65600040 100644 --- a/client/galaxy/scripts/components/RuleBuilder/RegularExpressionInput.vue +++ b/client/galaxy/scripts/components/RuleBuilder/RegularExpressionInput.vue @@ -2,7 +2,15 @@
- +
@@ -29,7 +37,7 @@ export default { return _l( `Regular expressions are patterns used to match character combinations in strings. This input accepts Python-style regular expressions, find more information about these in this Python for Biologists tutorial.` ); - } - } -} - \ No newline at end of file + }, + }, +}; + diff --git a/client/galaxy/scripts/components/RuleBuilder/RuleComponent.vue b/client/galaxy/scripts/components/RuleBuilder/RuleComponent.vue index cbeacf29977..474e6cbc4a4 100644 --- a/client/galaxy/scripts/components/RuleBuilder/RuleComponent.vue +++ b/client/galaxy/scripts/components/RuleBuilder/RuleComponent.vue @@ -2,8 +2,8 @@
- - + +
@@ -44,5 +44,5 @@ export default { return "rule-edit-" + this.ruleType.replace(/_/g, "-"); }, }, -} +}; diff --git a/client/galaxy/scripts/components/RuleBuilder/RuleDisplay.vue b/client/galaxy/scripts/components/RuleBuilder/RuleDisplay.vue index d3188159d6c..1d52daf8011 100644 --- a/client/galaxy/scripts/components/RuleBuilder/RuleDisplay.vue +++ b/client/galaxy/scripts/components/RuleBuilder/RuleDisplay.vue @@ -1,4 +1,3 @@ - \ No newline at end of file + diff --git a/client/galaxy/scripts/components/RuleBuilder/RuleModalHeader.vue b/client/galaxy/scripts/components/RuleBuilder/RuleModalHeader.vue index ccd1c57734f..9ea12aefa61 100644 --- a/client/galaxy/scripts/components/RuleBuilder/RuleModalHeader.vue +++ b/client/galaxy/scripts/components/RuleBuilder/RuleModalHeader.vue @@ -1,3 +1,3 @@ \ No newline at end of file + diff --git a/client/galaxy/scripts/components/RuleBuilder/RuleModalMiddle.vue b/client/galaxy/scripts/components/RuleBuilder/RuleModalMiddle.vue index 745fa17ceff..35fca19d00a 100644 --- a/client/galaxy/scripts/components/RuleBuilder/RuleModalMiddle.vue +++ b/client/galaxy/scripts/components/RuleBuilder/RuleModalMiddle.vue @@ -1,3 +1,3 @@ \ No newline at end of file + diff --git a/client/galaxy/scripts/components/RuleBuilder/RuleTargetComponent.vue b/client/galaxy/scripts/components/RuleBuilder/RuleTargetComponent.vue index ef622ae0b28..aeb4d07d1df 100644 --- a/client/galaxy/scripts/components/RuleBuilder/RuleTargetComponent.vue +++ b/client/galaxy/scripts/components/RuleBuilder/RuleTargetComponent.vue @@ -1,5 +1,11 @@ \ No newline at end of file + }, +}; + diff --git a/client/galaxy/scripts/components/RuleBuilder/StateDiv.vue b/client/galaxy/scripts/components/RuleBuilder/StateDiv.vue index c7cc9eb1292..2f06d8daf7f 100644 --- a/client/galaxy/scripts/components/RuleBuilder/StateDiv.vue +++ b/client/galaxy/scripts/components/RuleBuilder/StateDiv.vue @@ -2,4 +2,4 @@
- \ No newline at end of file + diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 5f4098da3e6..ee2162d7cb8 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -587,7 +587,6 @@ import RuleModalMiddle from "components/RuleBuilder/RuleModalMiddle"; import RuleModalFooter from "components/RuleBuilder/RuleModalFooter"; import StateDiv from "components/RuleBuilder/StateDiv"; - Vue.use(BootstrapVue); const RULES = RuleDefs.RULES; From 7bcd617c9f8558f5292ca369da596b329a4db306 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Sat, 28 Mar 2020 14:38:38 -0400 Subject: [PATCH 234/324] Change session timeout redirect URL to login entrypoint. --- lib/galaxy/web/framework/webapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index d77b230a275..179e6bd41f0 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -251,7 +251,7 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, self.user = None self.galaxy_session = None else: - self.response.send_redirect(url_for(controller='user', + self.response.send_redirect(url_for(controller='root', action='login', message="You have been logged out due to inactivity. Please log in again to continue using Galaxy.", status='info', From 5ca5d015c944ba5694ebc1c95c5b73e70333bfb9 Mon Sep 17 00:00:00 2001 From: assuntad23 Date: Fri, 17 Apr 2020 12:47:43 -0400 Subject: [PATCH 235/324] Changed CSS overflow property from scroll to hidden to make table scrollable --- .../galaxy/scripts/components/RuleCollectionBuilder.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index d4d99436d37..f8bb6d6696d 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -446,7 +446,7 @@ :data="hotData.data" :col-headers="colHeadersDisplay" :read-only="true" - stretch-h="all" + :stretch-h="all" >
@@ -1977,19 +1977,19 @@ export default {