From be7628531927ca826e1b0d05f435da19e721ea8d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 21 Jan 2019 16:50:09 +0000 Subject: [PATCH 01/21] Remove unneeded conditional requirements They are already included in `requirements.txt` . `ConditionalDependencies.check_python_openid()` was already removed in commit f31d61b71c739c03e5068bff188a56f175a18297 . --- lib/galaxy/dependencies/__init__.py | 3 --- lib/galaxy/dependencies/conditional-requirements.txt | 2 -- 2 files changed, 5 deletions(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 953aac5bf7d..bda5016208d 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -130,9 +130,6 @@ class ConditionalDependencies(object): def check_azure_storage(self): return 'azure_blob' in self.object_stores - def check_cloudbridge(self): - return 'cloud' in self.object_stores - def check_kamaki(self): return 'pithos' in self.object_stores diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 92765c5d867..662871a7b51 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -1,7 +1,6 @@ # These dependencies are only required when certain config options are set psycopg2-binary==2.7.4 weberror==0.10.3 -python-openid mysql-python fluent-logger raven @@ -13,7 +12,6 @@ azure-storage==0.32.0 # PyRods not in PyPI python-ldap==2.4.44 python-pam -cloudbridge==1.0.1 galaxycloudrunner # Chronos client From d0373c6a956cea15618db2021f3501117078a23f Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 21 Jan 2019 17:40:58 +0000 Subject: [PATCH 02/21] ``make config-rebuild`` Follow-up on https://github.com/galaxyproject/galaxy/pull/7028 . --- config/galaxy.yml.sample | 15 +------------ config/tool_shed.yml.sample | 4 ---- doc/source/admin/galaxy_options.rst | 35 +---------------------------- 3 files changed, 2 insertions(+), 52 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 06a29868639..8a68c6b5971 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -599,8 +599,7 @@ galaxy: # Activation grace period (in hours). Activation is not forced (login # is not disabled) until grace period has passed. Users under grace - # period can't run jobs. Enter 0 to disable grace period. Users with - # OpenID logins have grace period forever. + # period can't run jobs. Enter 0 to disable grace period. #activation_grace_period: 3 # Shown in warning box to users that were not activated yet. In use @@ -1428,18 +1427,6 @@ galaxy: # particular history #history_local_serial_workflow_scheduling: false - # Enable authentication via OpenID. Allows users to log in to their - # Galaxy account by authenticating with an OpenID provider. - #enable_openid: false - - # If OpenID is enabled, this configuration file specifies providers to - # use. Falls back to the .sample variant in config if default does not - # exist. - #openid_config_file: config/openid_conf.xml - - # If OpenID is enabled, consumer cache directory to use. - #openid_consumer_cache_path: database/openid_consumer_cache - # Enables and disables OpenID Connect (OIDC) support. #enable_oidc: false diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample index 7b4b90a7412..c647b7743fd 100644 --- a/config/tool_shed.yml.sample +++ b/config/tool_shed.yml.sample @@ -351,10 +351,6 @@ tool_shed: # will not clobber each other. #cookie_path: null - # Enable authentication via OpenID. Allows users to log in to their - # Galaxy account by authenticating with an OpenID provider. - #enable_openid: false - # Turn on logging of user actions to the database. Actions currently # logged are grid views, tool searches, and use of "recently" used # tools menu. The log_events and log_actions functionality will diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 603ddb62894..cdcfc687001 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1107,7 +1107,7 @@ Activation grace period (in hours). Activation is not forced (login is not disabled) until grace period has passed. Users under grace period can't run jobs. Enter 0 to disable grace - period. Users with OpenID logins have grace period forever. + period. :Default: ``3`` :Type: int @@ -2952,39 +2952,6 @@ :Type: bool -~~~~~~~~~~~~~~~~~ -``enable_openid`` -~~~~~~~~~~~~~~~~~ - -:Description: - Enable authentication via OpenID. Allows users to log in to their - Galaxy account by authenticating with an OpenID provider. -:Default: ``false`` -:Type: bool - - -~~~~~~~~~~~~~~~~~~~~~~ -``openid_config_file`` -~~~~~~~~~~~~~~~~~~~~~~ - -:Description: - If OpenID is enabled, this configuration file specifies providers - to use. Falls back to the .sample variant in config if default - does not exist. -:Default: ``config/openid_conf.xml`` -:Type: str - - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``openid_consumer_cache_path`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:Description: - If OpenID is enabled, consumer cache directory to use. -:Default: ``database/openid_consumer_cache`` -:Type: str - - ~~~~~~~~~~~~~~~ ``enable_oidc`` ~~~~~~~~~~~~~~~ From d5b306dd216e1b8d0f2ed34c52e562df643c8069 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 21 Jan 2019 17:44:32 +0000 Subject: [PATCH 03/21] Remove OpenID auth from docs Follow-up on https://github.com/galaxyproject/galaxy/pull/7028 . --- doc/source/admin/authentication.md | 11 ----------- doc/source/admin/config.rst | 2 -- doc/source/lib/galaxy.openid.rst | 20 -------------------- doc/source/lib/galaxy.rst | 1 - doc/source/lib/galaxy.web.framework.rst | 8 -------- 5 files changed, 42 deletions(-) delete mode 100644 doc/source/lib/galaxy.openid.rst diff --git a/doc/source/admin/authentication.md b/doc/source/admin/authentication.md index 2d22d4a3402..bae80da8d92 100644 --- a/doc/source/admin/authentication.md +++ b/doc/source/admin/authentication.md @@ -5,7 +5,6 @@ Galaxy supports the following authentication mechanisms: * [Galaxy Database](#galaxy-database) - Galaxy-specific login using e-mail address and password (the default); * [OIDC and OAuth2.0](#OIDC-and-OAuth2.0) - Login to Galaxy using your Google account, without having to create a Galaxy user; * [Authentication Framework](#authentication-framework) - A plugin-driven framework supporting LDAP/Active Directory and PAM; -* [OpenID](#openid) - authentication with Galaxy as a relying party; * [Proxy Authentication](#proxy_authentication) - HTTP [remote user](http://httpd.apache.org/docs/current/mod/mod_cgi.html#env) provided by any front-end Web server. ## Galaxy Database @@ -38,16 +37,6 @@ To configure one or more authentication plugins, simply copy ``config/auth_conf. The provided sample configuration file has numerous commented out examples and serves as the most up-to-date source of documentation on configuring these plugins. -## OpenID - -[OpenID](https://en.wikipedia.org/wiki/OpenID) is becoming less popular and probably shouldn't be used the primary mechanism -for authentication in Galaxy but it is an available option. - -Enabling OpenID requires you to edit Galaxy's configuration file and set `enable_openid` to `true`. This file is -likely located in `config/galaxy.yml` and can be created by copying Galaxy's sample `config/galaxy.yml.sample`. - -Enabling this option enables OpenID and causes the OpenID form to be displayed on the login screen. - ## Remote User Authentication If Galaxy is deployed with either nginx or Apache serving as a front-end proxy for Galaxy requests, they can be configured diff --git a/doc/source/admin/config.rst b/doc/source/admin/config.rst index c34cd65e66b..b10be4dc053 100644 --- a/doc/source/admin/config.rst +++ b/doc/source/admin/config.rst @@ -75,8 +75,6 @@ Additional configuration files and their purposes are: used by users to control runtime parameters such as memory allocations, cluster selection, and so forth. - ``object_store_conf.xml``: Configures more advanced storage paradigms for Galaxy datasets, including layout across multiple filesystems, or in object storage systems such as Swift or Amazon S3. -- ``openid_conf.xml``: Controls which OpenID (if enabled) providers should be presented as options to the user on the - login form. - ``swarm_manager_conf.yml``: Configures the experimental Docker Swarm manager. - ``tool_destinations.yml``: Configures dynamic tool destinations, which allow for mapping tools to job destinations based on certain runtime job properties, such as the user submitting it, input sizes, and so forth. diff --git a/doc/source/lib/galaxy.openid.rst b/doc/source/lib/galaxy.openid.rst deleted file mode 100644 index c110ecd22a8..00000000000 --- a/doc/source/lib/galaxy.openid.rst +++ /dev/null @@ -1,20 +0,0 @@ -galaxy\.openid package -====================== - -.. automodule:: galaxy.openid - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -galaxy\.openid\.providers module --------------------------------- - -.. automodule:: galaxy.openid.providers - :members: - :undoc-members: - :show-inheritance: - - diff --git a/doc/source/lib/galaxy.rst b/doc/source/lib/galaxy.rst index 4e932d7e9fb..95fbf6de2db 100644 --- a/doc/source/lib/galaxy.rst +++ b/doc/source/lib/galaxy.rst @@ -25,7 +25,6 @@ Subpackages galaxy.managers galaxy.model galaxy.objectstore - galaxy.openid galaxy.quota galaxy.security galaxy.tools diff --git a/doc/source/lib/galaxy.web.framework.rst b/doc/source/lib/galaxy.web.framework.rst index 97becf31bba..6aba7ac3035 100644 --- a/doc/source/lib/galaxy.web.framework.rst +++ b/doc/source/lib/galaxy.web.framework.rst @@ -33,14 +33,6 @@ galaxy\.web\.framework\.decorators module :undoc-members: :show-inheritance: -galaxy\.web\.framework\.openid\_manager module ----------------------------------------------- - -.. automodule:: galaxy.web.framework.openid_manager - :members: - :undoc-members: - :show-inheritance: - galaxy\.web\.framework\.webapp module ------------------------------------- From 31c71fbd6fb65cc4a4358e8b415d48885e73e7de Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Jan 2019 10:31:15 +0100 Subject: [PATCH 04/21] strip() text when loading xml Should fix https://github.com/galaxyproject/galaxy/issues/7250. --- lib/galaxy/util/__init__.py | 14 +++++++++- test/unit/test_utils.py | 54 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index ddfa7259751..52c2a3ce33d 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -222,6 +222,11 @@ def parse_xml(fname): tree = ElementTree.ElementTree() try: root = tree.parse(fname, parser=ElementTree.XMLParser(target=DoctypeSafeCallbackTarget())) + for elem in root.iter('*'): + if elem.text is not None: + elem.text = elem.text.strip() + if elem.tail is not None: + elem.tail = elem.tail.strip() except ParseError: log.exception("Error parsing file %s", fname) raise @@ -231,11 +236,18 @@ def parse_xml(fname): def parse_xml_string(xml_string): tree = ElementTree.fromstring(xml_string) + for elem in tree.iter('*'): + if elem.text is not None: + elem.text = elem.text.strip() + if elem.tail is not None: + elem.tail = elem.tail.strip() return tree def xml_to_string(elem, pretty=False): - """Returns a string from an xml tree""" + """ + Returns a string from an xml tree. + """ try: if elem is not None: if PY2: diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 536275c65e3..1c5ca1e5ac2 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -1,5 +1,17 @@ +from tempfile import NamedTemporaryFile + from galaxy import util +SECTION_XML = """ +
+ + + toolshed.g2.bx.psu.edu + + +
+""" + def test_strip_control_characters(): s = '\x00bla' @@ -15,3 +27,45 @@ def test_strip_control_characters_nested(): assert util.strip_control_characters_nested(l)[0] == stripped_s assert util.strip_control_characters_nested(t)[0] == stripped_s assert util.strip_control_characters_nested(d)[42] == stripped_s + + +def test_parse_xml_string(): + section = util.parse_xml_string(SECTION_XML) + _verify_section(section) + + +def test_parse_xml_file(): + with NamedTemporaryFile(mode='w') as tmp: + tmp.write(SECTION_XML) + tmp.flush() + section = util.parse_xml(tmp.name).getroot() + _verify_section(section) + + +def _verify_section(section): + tool = next(iter(section)) + assert sorted(tool.items()) == [ + ('file', + 'toolshed.g2.bx.psu.edu/repos/peterjc/seq_filter_by_id/fb1313d79396/seq_filter_by_id/tools/seq_filter_by_id/seq_filter_by_id.xml'), + ('guid', + 'toolshed.g2.bx.psu.edu/repos/peterjc/seq_filter_by_id/seq_filter_by_id/0.2.5') + ] + assert next(iter(tool)).text == 'toolshed.g2.bx.psu.edu' + + +def test_xml_to_string(): + section = util.parse_xml_string(SECTION_XML) + s = util.xml_to_string(section) + assert len(s.split('\n')) == 1 + + +def test_xml_to_string_pretty(): + section = util.parse_xml_string(SECTION_XML) + s = util.xml_to_string(section, pretty=True) + PRETTY = """ +
+ + toolshed.g2.bx.psu.edu + +
""" + assert s == PRETTY From b123579a48823b912fdb03f85323525dc73b6da3 Mon Sep 17 00:00:00 2001 From: Mason Date: Tue, 22 Jan 2019 10:49:15 -0800 Subject: [PATCH 05/21] Changed logout behavior to automatically redirect home --- client/galaxy/scripts/layout/menu.js | 17 ++++++++++++++--- templates/user/logout.mako | 7 ++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js index 0a3e1b05827..62b7d78aa15 100644 --- a/client/galaxy/scripts/layout/menu.js +++ b/client/galaxy/scripts/layout/menu.js @@ -9,6 +9,18 @@ import { CommunicationServerView } from "layout/communication-server-view"; import Webhooks from "mvc/webhooks"; import Utils from "utils/utils"; + +function logoutClick() { + let galaxy = getGalaxyInstance(); + let token = galaxy.session_csrf_token || ""; + if (galaxy.user) { + galaxy.user.clearSessionStorage(); + } + let url = `${galaxy.root}user/logout?session_csrf_token=${token}`; + window.top.location.href = url; +} + + var Collection = Backbone.Collection.extend({ model: Backbone.Model.extend({ defaults: { @@ -290,9 +302,8 @@ var Collection = Backbone.Collection.extend({ }, { title: _l("Logout"), - url: `user/logout?session_csrf_token=${Galaxy.session_csrf_token}`, - target: "_top", - divider: true + divider: true, + onclick: logoutClick }, { title: _l("Saved Datasets"), diff --git a/templates/user/logout.mako b/templates/user/logout.mako index 83020695a62..20a38c04f36 100644 --- a/templates/user/logout.mako +++ b/templates/user/logout.mako @@ -38,11 +38,8 @@ def inherit(context): <%def name="javascript_app()"> ${ parent.javascript_app() } \ No newline at end of file From eb406cc210b556a9218d778383451916d7cd1ecf Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Jan 2019 13:51:03 +0100 Subject: [PATCH 06/21] Fix install operation in installed_repository_grid The incoming url keywords look like this: ``` operation=install&id=ebfb8f50c6abde6d?async=false&sort=name&page=1&show_item_checkboxes=true&advanced_search=false&operation=install&id=ebfb8f50c6abde6d ``` and the id parameter gets parsed into a list: ``` ['ebfb8f50c6abde6d?async=false', 'ebfb8f50c6abde6d'] ``` Obviously there's something wrong in the grid or mako, but this works fine. Fixes https://github.com/galaxyproject/galaxy/issues/5619 --- lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 18ec840efbf..cbc6caee9f8 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -587,6 +587,11 @@ class AdminToolshed(AdminGalaxy): repository_id = kwd.get('id', None) if repository_id is None: return trans.show_error_message('Missing required encoded repository id.') + if repository_id and isinstance(repository_id, list): + # FIXME: This is a hack that avoids unhandled and duplicate url parameters leaking in. + # This should be handled somewhere in the grids system, but given the legacy status + # this should be OK. + repository_id = [r for r in repository_id if '=' not in r][0] # This method only work for a single repo id operation = kwd.get('operation', None) repository = repository_util.get_installed_tool_shed_repository(trans.app, repository_id) if repository is None: From 35fe3bf716e4f527755f75ebb99b9dbdb777e37d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 23 Jan 2019 16:00:50 +0100 Subject: [PATCH 07/21] Not all workflow outputs have a UUID --- lib/galaxy/workflow/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index db7396136a6..4b09688acdb 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -354,7 +354,7 @@ class SubWorkflowModule(WorkflowModule): if not label: label = "%s:%s" % (order_index, workflow_output['output_name']) for data_output in data_outputs: - if data_output['name'] == workflow_output['output_name'] or data_output['uuid'] == workflow_output['uuid']: + if data_output['name'] == workflow_output['output_name'] or data_output.get('uuid', object()) == workflow_output.get('uuid', object()): data_output['label'] = label data_output['name'] = label # That's the right data_output From d95ba26a24ce2960bc565f5a021338a9ec68d061 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 23 Jan 2019 19:47:05 +0100 Subject: [PATCH 08/21] Fix saving imported subworkflows whose child-workflow is included more than once. Fixes https://github.com/galaxyproject/galaxy/issues/7261 and probably https://github.com/galaxyproject/galaxy/issues/6032 --- lib/galaxy/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 001cd2a12e4..5ac77e02010 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3881,7 +3881,7 @@ class Workflow(Dictifiable): top_level_workflow = self if self.stored_workflow is None: # TODO: enforce this at creation... - assert len(self.parent_workflow_steps) == 1 + assert len(set(w.uuid for w in self.parent_workflow_steps)) == 1 return self.parent_workflow_steps[0].workflow.top_level_workflow return top_level_workflow From f2a454c776bea547967c867e47a827a9a969f67c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 23 Jan 2019 19:29:37 +0100 Subject: [PATCH 09/21] Fix output uuid comparison --- lib/galaxy/workflow/modules.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 4b09688acdb..a75ef714f7a 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -353,8 +353,10 @@ class SubWorkflowModule(WorkflowModule): label = workflow_output['label'] if not label: label = "%s:%s" % (order_index, workflow_output['output_name']) + workflow_output_uuid = workflow_output.get('uuid') or object() for data_output in data_outputs: - if data_output['name'] == workflow_output['output_name'] or data_output.get('uuid', object()) == workflow_output.get('uuid', object()): + data_output_uuid = data_output.get('uuid') or object() + if data_output['name'] == workflow_output['output_name'] or data_output_uuid == workflow_output_uuid: data_output['label'] = label data_output['name'] = label # That's the right data_output From 5adfa81a8b6366467d003a9e7b2a551255fdfb20 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 24 Jan 2019 10:56:40 +0000 Subject: [PATCH 10/21] Update sqlalchemy-migrate requirement to 0.12.0 Fix https://github.com/galaxyproject/galaxy/issues/7159 --- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 434ff91cfce..db2360636a3 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -136,7 +136,7 @@ s3transfer==0.1.13 simplejson==3.16.0 six==1.11.0 social-auth-core==1.5.0 -sqlalchemy-migrate==0.11.0 +sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.33.3 sqlalchemy==1.2.11 sqlparse==0.2.4 From 687c99c0094d1ff0ee91ef581fb340a7f09e2404 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 24 Jan 2019 12:04:31 +0000 Subject: [PATCH 11/21] Remove duplicated entry in ``.gitignore`` --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9bd8443d03b..41025faaf15 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,7 @@ test/unit/**.log # Project files *.kpf +.idea .vscode client/**/jsconfig.json @@ -152,5 +153,3 @@ client/webpack-stats.json .DS_Store *.rej *~ -.idea -.vscode From 9207b56eebaa06eed0614fdb877abc9b6744cf44 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 24 Jan 2019 12:09:33 +0000 Subject: [PATCH 12/21] Fix "occured" typo everywhere Fix https://github.com/galaxyproject/galaxy/issues/7264 --- .../galaxy/scripts/mvc/dataset/dataset-edit-attributes.js | 4 ++-- client/galaxy/scripts/mvc/dataset/dataset-error.js | 8 ++++---- client/galaxy/scripts/mvc/library/library-dataset-view.js | 4 ++-- .../galaxy/scripts/mvc/library/library-folderrow-view.js | 6 +++--- .../scripts/mvc/library/library-foldertoolbar-view.js | 4 ++-- .../galaxy/scripts/mvc/library/library-libraryrow-view.js | 6 +++--- .../scripts/mvc/library/library-librarytoolbar-view.js | 2 +- lib/galaxy/datatypes/isa.py | 2 +- lib/galaxy/jobs/runners/drmaa.py | 4 ++-- lib/galaxy/managers/datasets.py | 4 ++-- lib/galaxy/managers/jobs.py | 2 +- lib/galaxy/util/filelock.py | 2 +- lib/galaxy/webapps/galaxy/api/authenticate.py | 2 +- lib/galaxy/webapps/galaxy/api/libraries.py | 2 +- lib/galaxy/webapps/galaxy/api/library_datasets.py | 4 ++-- tools/filters/sff_extract.py | 2 +- 16 files changed, 29 insertions(+), 29 deletions(-) diff --git a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js index f80d1058f59..fca53d24b16 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js @@ -48,7 +48,7 @@ var View = Backbone.View.extend({ var err_msg = response.responseJSON && response.responseJSON.err_msg; self.message.update({ status: "danger", - message: err_msg || "Error occured while loading the dataset." + message: err_msg || "Error occurred while loading the dataset." }); } }); @@ -73,7 +73,7 @@ var View = Backbone.View.extend({ var err_msg = response.responseJSON && response.responseJSON.err_msg; self.message.update({ status: "danger", - message: err_msg || "Error occured while editing the dataset attributes." + message: err_msg || "Error occurred while editing the dataset attributes." }); } }); diff --git a/client/galaxy/scripts/mvc/dataset/dataset-error.js b/client/galaxy/scripts/mvc/dataset/dataset-error.js index 33e20b5b0d3..15ed502ce27 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-error.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-error.js @@ -36,7 +36,7 @@ var View = Backbone.View.extend({ error: response => { var error_response = { status: "error", - message: "Error occured while loading the job.", + message: "Error occurred while loading the job.", persistent: true, cls: "errormessage" }; @@ -47,7 +47,7 @@ var View = Backbone.View.extend({ error: response => { var error_response = { status: "error", - message: "Error occured while loading the dataset.", + message: "Error occurred while loading the dataset.", persistent: true, cls: "errormessage" }; @@ -61,7 +61,7 @@ var View = Backbone.View.extend({ this.$el.empty().append(` ${this._templateHeader()}

Dataset Error

-

An error occured while running the tool ${job.tool_id}.

+

An error occurred while running the tool ${job.tool_id}.

Tool execution generated the following messages:

${_.escape(job.stderr)}
@@ -194,7 +194,7 @@ var View = Backbone.View.extend({ error: response => { var error_response = { status: "error", - message: "Error occured while saving. Please fill all the required fields and try again.", + message: "Error occurred while saving. Please fill all the required fields and try again.", persistent: true, cls: "errormessage" }; diff --git a/client/galaxy/scripts/mvc/library/library-dataset-view.js b/client/galaxy/scripts/mvc/library/library-dataset-view.js index 1869f0666ba..bb68e6cbd03 100644 --- a/client/galaxy/scripts/mvc/library/library-dataset-view.js +++ b/client/galaxy/scripts/mvc/library/library-dataset-view.js @@ -276,7 +276,7 @@ var LibraryDatasetView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(`Dataset not imported. ${response.responseJSON.err_msg}`); } else { - mod_toastr.error("An error occured. Dataset not imported. Please try again."); + mod_toastr.error("An error occurred. Dataset not imported. Please try again."); } } } @@ -446,7 +446,7 @@ var LibraryDatasetView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(response.responseJSON.err_msg); } else { - mod_toastr.error("An error occured while attempting to update the library dataset."); + mod_toastr.error("An error occurred while attempting to update the library dataset."); } } }); diff --git a/client/galaxy/scripts/mvc/library/library-folderrow-view.js b/client/galaxy/scripts/mvc/library/library-folderrow-view.js index 5398a5260f9..adb5190c105 100644 --- a/client/galaxy/scripts/mvc/library/library-folderrow-view.js +++ b/client/galaxy/scripts/mvc/library/library-folderrow-view.js @@ -136,7 +136,7 @@ var FolderRowView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(`Dataset was not undeleted. ${response.responseJSON.err_msg}`); } else { - mod_toastr.error("An error occured! Dataset was not undeleted. Please try again."); + mod_toastr.error("An error occurred! Dataset was not undeleted. Please try again."); } } }); @@ -163,7 +163,7 @@ var FolderRowView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(`Folder was not undeleted. ${response.responseJSON.err_msg}`); } else { - mod_toastr.error("An error occured! Folder was not undeleted. Please try again."); + mod_toastr.error("An error occurred! Folder was not undeleted. Please try again."); } } }); @@ -213,7 +213,7 @@ var FolderRowView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(response.responseJSON.err_msg); } else { - mod_toastr.error("An error occured while attempting to update the folder."); + mod_toastr.error("An error occurred while attempting to update the folder."); } } }); diff --git a/client/galaxy/scripts/mvc/library/library-foldertoolbar-view.js b/client/galaxy/scripts/mvc/library/library-foldertoolbar-view.js index 4176034ce73..b3c46b63608 100644 --- a/client/galaxy/scripts/mvc/library/library-foldertoolbar-view.js +++ b/client/galaxy/scripts/mvc/library/library-foldertoolbar-view.js @@ -885,7 +885,7 @@ var FolderToolbarView = Backbone.View.extend({ mod_toastr.success("Selected files imported into the current folder"); Galaxy.modal.hide(); } else { - mod_toastr.error("An error occured."); + mod_toastr.error("An error occurred."); } return true; } @@ -934,7 +934,7 @@ var FolderToolbarView = Backbone.View.extend({ Galaxy.modal.hide(); } else { // TODO better error report - mod_toastr.error("An error occured."); + mod_toastr.error("An error occurred."); } return true; } diff --git a/client/galaxy/scripts/mvc/library/library-libraryrow-view.js b/client/galaxy/scripts/mvc/library/library-libraryrow-view.js index fbead05760d..a17ee1c1d9a 100644 --- a/client/galaxy/scripts/mvc/library/library-libraryrow-view.js +++ b/client/galaxy/scripts/mvc/library/library-libraryrow-view.js @@ -162,7 +162,7 @@ var LibraryRowView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(response.responseJSON.err_msg); } else { - mod_toastr.error("An error occured while attempting to update the library."); + mod_toastr.error("An error occurred while attempting to update the library."); } } }); @@ -197,7 +197,7 @@ var LibraryRowView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(response.responseJSON.err_msg); } else { - mod_toastr.error("An error occured during deleting the library."); + mod_toastr.error("An error occurred during deleting the library."); } } }); @@ -224,7 +224,7 @@ var LibraryRowView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(response.responseJSON.err_msg); } else { - mod_toastr.error("An error occured while undeleting the library."); + mod_toastr.error("An error occurred while undeleting the library."); } } }); diff --git a/client/galaxy/scripts/mvc/library/library-librarytoolbar-view.js b/client/galaxy/scripts/mvc/library/library-librarytoolbar-view.js index d0e7a1d69e2..a42872810d3 100644 --- a/client/galaxy/scripts/mvc/library/library-librarytoolbar-view.js +++ b/client/galaxy/scripts/mvc/library/library-librarytoolbar-view.js @@ -112,7 +112,7 @@ var LibraryToolbarView = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { mod_toastr.error(response.responseJSON.err_msg); } else { - mod_toastr.error("An error occured."); + mod_toastr.error("An error occurred."); } } }); diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py index 0089fd326f6..d11f6454612 100644 --- a/lib/galaxy/datatypes/isa.py +++ b/lib/galaxy/datatypes/isa.py @@ -271,7 +271,7 @@ class _Isa(data.Data): if investigation is None: html = """
Error while reading ISA archive.
-

An error occured while reading content of ISA archive.

+

An error occurred while reading content of ISA archive.

If you have tried to load your archive with the uploader by selecting isa-tab as composite data type, then try to load it again with isa-json instead. Conversely, if you have tried to load your archive with the uploader by selecting isa-json as composite data type, then try isa-tab instead.

You may also try to look into your zip file in order to find out if this is a proper ISA archive. If you see a file i_Investigation.txt inside, then it is an ISA-Tab archive. If you see a file with extension .json inside, then it is an ISA-JSON archive. If you see nothing like that, then either your ISA archive is corrupted, or it is not an ISA archive.

""" diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 2d549f48d9b..ab465f34d3c 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -254,11 +254,11 @@ class DRMAAJobRunner(AsynchronousJobRunner): """ look at a single watched job, determine its state, and deal with errors that could happen in this process. to be called from check_watched_items() - returns the state or None if exceptions occured + returns the state or None if exceptions occurred in the latter case the job is appended to new_watched if a 1 drmaa.InternalException, 2 drmaa.InvalidJobExceptionnot, or - 3 drmaa.DrmCommunicationException occured + 3 drmaa.DrmCommunicationException occurred (which causes the job to be tested again in the next iteration of check_watched_items) - the job is finished as errored if any other exception occurs - the job is finished OK or errored after the maximum number of retries diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 93318c38c1a..56d1c77e357 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -355,7 +355,7 @@ class DatasetAssociationManager(base.ModelManager, if action == 'remove_restrictions': trans.app.security_agent.make_dataset_public(dataset) if not trans.app.security_agent.dataset_is_public(dataset): - raise exceptions.InternalServerError('An error occured while making dataset public.') + raise exceptions.InternalServerError('An error occurred while making dataset public.') elif action == 'make_private': if not trans.app.security_agent.dataset_is_private_to_user(trans, dataset): private_role = trans.app.security_agent.get_private_user_role(trans.user) @@ -364,7 +364,7 @@ class DatasetAssociationManager(base.ModelManager, trans.sa_session.flush() if not trans.app.security_agent.dataset_is_private_to_user(trans, dataset): # Check again and inform the user if dataset is not private. - raise exceptions.InternalServerError('An error occured and the dataset is NOT private.') + raise exceptions.InternalServerError('An error occurred and the dataset is NOT private.') elif action == 'set_permissions': def to_role_id(encoded_role_id): diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index b7670902220..df0a92427e9 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -153,7 +153,7 @@ class JobSearch(object): # We need to make sure that the job we are looking for has been run with identical inputs. # Here we deal with 3 requirements: # - the jobs' input dataset (=b) version is 0, meaning the job's input dataset is not yet ready - # - b's update_time is older than the job create time, meaning no changes occured + # - b's update_time is older than the job create time, meaning no changes occurred # - the job has a dataset_version recorded, and that versions' metadata matches c's metadata. or_( and_(or_(a.dataset_version.in_([0, b.version]), diff --git a/lib/galaxy/util/filelock.py b/lib/galaxy/util/filelock.py index be6dc512b83..e6edcae9281 100644 --- a/lib/galaxy/util/filelock.py +++ b/lib/galaxy/util/filelock.py @@ -46,7 +46,7 @@ class FileLock(object): if e.errno != errno.EEXIST: raise if (time.time() - start_time) >= self.timeout: - raise FileLockException("Timeout occured.") + raise FileLockException("Timeout occurred.") time.sleep(self.delay) self.is_locked = True diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index ffa420396d8..6e032015369 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -53,7 +53,7 @@ class AuthenticationController(BaseAPIController): raise exceptions.ObjectNotFound('The user does not exist.') elif len(user) > 1: # DB is inconsistent and we have more users with the same email. - raise exceptions.InconsistentDatabase('An error occured, please contact your administrator.') + raise exceptions.InconsistentDatabase('An error occurred, please contact your administrator.') else: user = user[0] is_valid_user = self.app.auth_manager.check_password(user, password) diff --git a/lib/galaxy/webapps/galaxy/api/libraries.py b/lib/galaxy/webapps/galaxy/api/libraries.py index 6ad873ef471..3ce60eaaa00 100644 --- a/lib/galaxy/webapps/galaxy/api/libraries.py +++ b/lib/galaxy/webapps/galaxy/api/libraries.py @@ -295,7 +295,7 @@ class LibrariesController(BaseAPIController): elif action == 'remove_restrictions': is_public = self.library_manager.make_public(trans, library) if not is_public: - raise exceptions.InternalServerError('An error occured while making library public.') + raise exceptions.InternalServerError('An error occurred while making library public.') elif action == 'set_permissions': # ACCESS LIBRARY ROLES diff --git a/lib/galaxy/webapps/galaxy/api/library_datasets.py b/lib/galaxy/webapps/galaxy/api/library_datasets.py index cb4d2eec9e2..cbc6352c60f 100644 --- a/lib/galaxy/webapps/galaxy/api/library_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/library_datasets.py @@ -230,7 +230,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra if action == 'remove_restrictions': trans.app.security_agent.make_dataset_public(dataset) if not trans.app.security_agent.dataset_is_public(dataset): - raise exceptions.InternalServerError('An error occured while making dataset public.') + raise exceptions.InternalServerError('An error occurred while making dataset public.') elif action == 'make_private': if not trans.app.security_agent.dataset_is_private_to_user(trans, dataset): private_role = trans.app.security_agent.get_private_user_role(trans.user) @@ -239,7 +239,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra trans.sa_session.flush() if not trans.app.security_agent.dataset_is_private_to_user(trans, dataset): # Check again and inform the user if dataset is not private. - raise exceptions.InternalServerError('An error occured and the dataset is NOT private.') + raise exceptions.InternalServerError('An error occurred and the dataset is NOT private.') elif action == 'set_permissions': # ACCESS DATASET ROLES valid_access_roles = [] diff --git a/tools/filters/sff_extract.py b/tools/filters/sff_extract.py index 98620572776..702cfee3f63 100644 --- a/tools/filters/sff_extract.py +++ b/tools/filters/sff_extract.py @@ -1139,7 +1139,7 @@ def launch_ssaha(linker_fname, query_fname, output_fh): print("ok.") except Exception: print("\n") - raise RuntimeError('An error occured during the SSAHA2 execution, aborting.') + raise RuntimeError('An error occurred during the SSAHA2 execution, aborting.') def read_ssaha_data(ssahadata_fh): From 12128ad49d763ad725e5861ff0e1cd5aaeb5461e Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 24 Jan 2019 13:37:24 -0500 Subject: [PATCH 13/21] Fix _.escape typo in toolshed group detail view --- client/galaxy/scripts/toolshed/groups/group-detail-view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/toolshed/groups/group-detail-view.js b/client/galaxy/scripts/toolshed/groups/group-detail-view.js index 869551e5bf1..6588dc12048 100644 --- a/client/galaxy/scripts/toolshed/groups/group-detail-view.js +++ b/client/galaxy/scripts/toolshed/groups/group-detail-view.js @@ -95,7 +95,7 @@ const GroupDetailView = Backbone.View.extend({ '<% _.each(group.get("repositories"), function(repo) { %>', "", "", - '<%= _.escape(repo.name) %>', + '<%= _.escape(repo.name) %>', "", "", "<%= _.escape(repo.description) %>", From d83db6a53796137c8ccc96efe3b40f3f3f88f694 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 27 Jan 2019 15:48:16 -0500 Subject: [PATCH 14/21] Stop with tool migrations by default, they are not needed in modern Galaxies... ... and even problematic in some ways. --- config/galaxy.yml.sample | 11 +++++------ doc/source/admin/galaxy_options.rst | 13 ++++++------- lib/galaxy/config.py | 2 +- lib/galaxy/webapps/galaxy/config_schema.yml | 10 ++++------ 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index f2f30ce7bee..a0258b4a095 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -181,12 +181,11 @@ galaxy: # Enable / disable checking if any tools defined in the above non-shed # tool_config_files (i.e., tool_conf.xml) have been migrated from the - # Galaxy code distribution to the Tool Shed. This setting should - # generally be set to False only for development Galaxy environments - # that are often rebuilt from scratch where migrated tools do not need - # to be available in the Galaxy tool panel. If the following setting - # remains commented, the default setting will be True. - #check_migrate_tools: true + # Galaxy code distribution to the Tool Shed. This functionality is + # largely untested in modern Galaxy releases and has serious issues + # such as #7273 and the possibility of slowing down Galaxy startup, so + # the default and recommended value is False. + #check_migrate_tools: false # Tool config maintained by tool migration scripts. If you use the # migration scripts to install tools that have been migrated to the diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index c9a2b47dc69..0d2d38e92c3 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -208,13 +208,12 @@ :Description: Enable / disable checking if any tools defined in the above non- shed tool_config_files (i.e., tool_conf.xml) have been migrated - from the Galaxy code distribution to the Tool Shed. This setting - should generally be set to False only for development Galaxy - environments that are often rebuilt from scratch where migrated - tools do not need to be available in the Galaxy tool panel. If - the following setting remains commented, the default setting will - be True. -:Default: ``true`` + from the Galaxy code distribution to the Tool Shed. This + functionality is largely untested in modern Galaxy releases and + has serious issues such as #7273 and the possibility of slowing + down Galaxy startup, so the default and recommended value is + False. +:Default: ``false`` :Type: bool diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index d243b59f352..8f0c1e128b7 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -240,7 +240,7 @@ class Configuration(object): # Check for tools defined in the above non-shed tool configs (i.e., tool_conf.xml) tht have # been migrated from the Galaxy code distribution to the Tool Shed. - self.check_migrate_tools = string_as_bool(kwargs.get('check_migrate_tools', True)) + self.check_migrate_tools = string_as_bool(kwargs.get('check_migrate_tools', False)) self.shed_tool_data_path = kwargs.get("shed_tool_data_path", None) self.x_frame_options = kwargs.get("x_frame_options", "SAMEORIGIN") if self.shed_tool_data_path: diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index ad44ea8d7c5..e58b7d22740 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -182,16 +182,14 @@ mapping: check_migrate_tools: type: bool - default: true + default: false required: false desc: | Enable / disable checking if any tools defined in the above non-shed tool_config_files (i.e., tool_conf.xml) have been migrated from the Galaxy - code distribution to the Tool Shed. This setting should generally be set to - False only for development Galaxy environments that are often rebuilt from - scratch where migrated tools do not need to be available in the Galaxy tool - panel. If the following setting remains commented, the default setting will - be True. + code distribution to the Tool Shed. This functionality is largely untested + in modern Galaxy releases and has serious issues such as #7273 and the possibility + of slowing down Galaxy startup, so the default and recommended value is False. migrated_tools_config: type: str From f05a3f9e5807a7b0a88232932cf29d40600aeb5a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 29 Jan 2019 10:23:39 +0100 Subject: [PATCH 15/21] Fix flakey history import metadata test We need to wait for the BAM dataset to be final before we can export. (At least that solved it locally for me). --- test/api/test_histories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api/test_histories.py b/test/api/test_histories.py index bf135b03504..51919e53902 100644 --- a/test/api/test_histories.py +++ b/test/api/test_histories.py @@ -192,7 +192,7 @@ class HistoriesApiTestCase(api.ApiTestCase): def test_import_metadata_regeneration(self): history_name = "for_import_metadata_regeneration" history_id = self.dataset_populator.new_history(name=history_name) - self.dataset_populator.new_dataset(history_id, content=open(self.test_data_resolver.get_filename("1.bam"), 'rb'), file_type='bam') + self.dataset_populator.new_dataset(history_id, content=open(self.test_data_resolver.get_filename("1.bam"), 'rb'), file_type='bam', wait=True) imported_history_id = self._reimport_history(history_id, history_name) self._assert_history_length(imported_history_id, 1) import_bam_metadata = self.dataset_populator.get_history_dataset_details( From 42052d6b8d62df625a099141364a2c5dad8ed0b2 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 29 Jan 2019 13:36:54 +0000 Subject: [PATCH 16/21] Do not skip the client build in dockerized selenium tests This allows the removal of the `docker run` of `make client-production-maps`, which was run on a docker image for the wrong version of `node`. --- .ci/jenkins/selenium-py3/run_tests.sh | 10 +------- .ci/jenkins/selenium/run_tests.sh | 10 +------- run_tests.sh | 37 +++++++++++++++++---------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.ci/jenkins/selenium-py3/run_tests.sh b/.ci/jenkins/selenium-py3/run_tests.sh index 0fb4805fdaf..827151835f6 100755 --- a/.ci/jenkins/selenium-py3/run_tests.sh +++ b/.ci/jenkins/selenium-py3/run_tests.sh @@ -2,7 +2,6 @@ # Enable retries on tests to reduce chances of transient failures. : ${GALAXY_TEST_SELENIUM_RETRIES:=1} -: ${GALAXY_TEST_CLIENT_BUILD_IMAGE:='node:9.4.0'} # If in Jenkins environment, use it for artifacts. if [ -n "$BUILD_NUMBER" ]; @@ -20,15 +19,8 @@ mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY" mkdir -p ~/.jenkins-yarn-cache YARN_CACHE_FOLDER=~/.jenkins-yarn-cache -# Set git environment variables to enable Git. https://github.com/galaxyproject/galaxy/issues/5912 -# Setup volume and environment variable to cache this users yarn build. -docker run -e GIT_COMMITTER_NAME=Jenkins -e GIT_COMMITTER_EMAIL=jenkins@galaxyproject.org \ - -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER:rw \ - -v `pwd`:`pwd`:rw -w `pwd` -u $UID $GALAXY_TEST_CLIENT_BUILD_IMAGE \ - /bin/bash -c 'make client-production-maps' - # Start Selenium server in the test Docker container. -DOCKER_RUN_EXTRA_ARGS="--shm-size=2g -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}" +DOCKER_RUN_EXTRA_ARGS="${DOCKER_RUN_EXTRA_ARGS} --shm-size=2g -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY}" export DOCKER_RUN_EXTRA_ARGS ./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --skip_flakey_fails --selenium "$@" diff --git a/.ci/jenkins/selenium/run_tests.sh b/.ci/jenkins/selenium/run_tests.sh index a5991b9685d..30840006815 100755 --- a/.ci/jenkins/selenium/run_tests.sh +++ b/.ci/jenkins/selenium/run_tests.sh @@ -2,7 +2,6 @@ # Enable retries on tests to reduce chances of transient failures. : ${GALAXY_TEST_SELENIUM_RETRIES:=1} -: ${GALAXY_TEST_CLIENT_BUILD_IMAGE:='node:9.4.0'} # If in Jenkins environment, use it for artifacts. if [ -n "$BUILD_NUMBER" ]; @@ -20,15 +19,8 @@ mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY" mkdir -p ~/.jenkins-yarn-cache YARN_CACHE_FOLDER=~/.jenkins-yarn-cache -# Set git environment variables to enable Git. https://github.com/galaxyproject/galaxy/issues/5912 -# Setup volume and environment variable to cache this users yarn build. -docker run -e GIT_COMMITTER_NAME=Jenkins -e GIT_COMMITTER_EMAIL=jenkins@galaxyproject.org \ - -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER:rw \ - -v `pwd`:`pwd`:rw -w `pwd` -u $UID $GALAXY_TEST_CLIENT_BUILD_IMAGE \ - /bin/bash -c 'make client-production-maps' - # Start Selenium server in the test Docker container. -DOCKER_RUN_EXTRA_ARGS="--shm-size=2g -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}" +DOCKER_RUN_EXTRA_ARGS="${DOCKER_RUN_EXTRA_ARGS} --shm-size=2g -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY}" export DOCKER_RUN_EXTRA_ARGS ./run_tests.sh --dockerize --db postgres --clean_pyc --skip_flakey_fails --selenium "$@" diff --git a/run_tests.sh b/run_tests.sh index 3ad62945d46..d565910a90b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -285,20 +285,31 @@ then DOCKER_EXTRA_ARGS=${DOCKER_ARGS:-""} DOCKER_RUN_EXTRA_ARGS=${DOCKER_RUN_EXTRA_ARGS:-""} DOCKER_IMAGE=${DOCKER_IMAGE:-${DOCKER_DEFAULT_IMAGE}} - if [ "$1" = "--python3" ]; then - DOCKER_RUN_EXTRA_ARGS="-e GALAXY_VIRTUAL_ENV=/galaxy_venv3 $DOCKER_RUN_EXTRA_ARGS" - shift 1 - fi - if [ "$1" = "--db" ]; then - db_type=$2 - shift 2 - else - db_type="sqlite" - fi + db_type="sqlite" + while [ $# -gt 0 ]; do + case "$1" in + --python3) + DOCKER_RUN_EXTRA_ARGS="${DOCKER_RUN_EXTRA_ARGS} -e GALAXY_VIRTUAL_ENV=/galaxy_venv3" + shift 1 + ;; + --db) + db_type=$2 + shift 2 + ;; + *) + break + ;; + esac + done + # Skip client build process in the Docker container for all tests except Selenium + GALAXY_SKIP_CLIENT_BUILD=1 + case "$*" in + *-selenium*) + GALAXY_SKIP_CLIENT_BUILD=0 + ;; + esac MY_UID=$(id -u) - # Skip client build process in the Docker container for all tests, the Jenkins task builds the client - # locally before testing - you will need to do this also if using this script for Selenium testing. - DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=1 ${DOCKER_RUN_EXTRA_ARGS}" + DOCKER_RUN_EXTRA_ARGS="${DOCKER_RUN_EXTRA_ARGS} -e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=${GALAXY_SKIP_CLIENT_BUILD}" echo "Docker version:" docker --version echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." From 53d2c3490adb54c73afc4397768f04999184b723 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 22 Jan 2019 15:15:55 -0500 Subject: [PATCH 17/21] Fix hotdata method access; this still needs a lot of love/refactoring, but this makes it work again? --- .../components/RuleCollectionBuilder.vue | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 7136d150a54..6d527df80e3 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -1257,8 +1257,9 @@ export default { return targets; }, colHeaders() { - const data = this.hotData["data"]; - const columns = this.hotData["columns"]; + let hotData = this.hotData(); + const data = hotData["data"]; + const columns = hotData["columns"]; return RuleDefs.colHeadersFor(data, columns); }, colHeadersDisplay() { @@ -1684,7 +1685,8 @@ export default { return identifierColumns; }, buildRequestElements(createDatasetDescription, createSubcollectionDescription, subElementProp) { - const data = this.hotData["data"]; + let hotData = this.hotData(); + const data = hotData["data"]; const identifierColumns = this.identifierColumns(); if (identifierColumns.length < 1) { console.log("Error but this shouldn't have happened, create button should have been disabled."); @@ -1791,8 +1793,9 @@ export default { return elementsByName; }, creationElementsFromDatasets() { - const sources = this.hotData["sources"]; - const data = this.hotData["data"]; + let hotData = this.hotData(); + const sources = hotData["sources"]; + const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; const elementsByCollectionName = this.buildRequestElements( @@ -1811,7 +1814,8 @@ export default { }, creationElementsForFetch() { // fetch elements for HDCA - const data = this.hotData["data"]; + let hotData = this.hotData(); + const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; const elementsByCollectionName = this.buildRequestElements( @@ -1830,7 +1834,8 @@ export default { }, creationDatasetsForFetch() { // fetch elements for HDAs if not collection information specified. - const data = this.hotData["data"]; + let hotData = this.hotData(); + const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; const datasets = []; From 96a25fbe31781f36f4607b374b238adbb5ed11ee Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 22 Jan 2019 17:12:38 -0500 Subject: [PATCH 18/21] Prefer const, none of these are reassigned. --- .../scripts/components/RuleCollectionBuilder.vue | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 6d527df80e3..2d9a8028969 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -1257,7 +1257,7 @@ export default { return targets; }, colHeaders() { - let hotData = this.hotData(); + const hotData = this.hotData(); const data = hotData["data"]; const columns = hotData["columns"]; return RuleDefs.colHeadersFor(data, columns); @@ -1685,7 +1685,7 @@ export default { return identifierColumns; }, buildRequestElements(createDatasetDescription, createSubcollectionDescription, subElementProp) { - let hotData = this.hotData(); + const hotData = this.hotData(); const data = hotData["data"]; const identifierColumns = this.identifierColumns(); if (identifierColumns.length < 1) { @@ -1793,7 +1793,7 @@ export default { return elementsByName; }, creationElementsFromDatasets() { - let hotData = this.hotData(); + const hotData = this.hotData(); const sources = hotData["sources"]; const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; @@ -1814,7 +1814,7 @@ export default { }, creationElementsForFetch() { // fetch elements for HDCA - let hotData = this.hotData(); + const hotData = this.hotData(); const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; @@ -1834,7 +1834,7 @@ export default { }, creationDatasetsForFetch() { // fetch elements for HDAs if not collection information specified. - let hotData = this.hotData(); + const hotData = this.hotData(); const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; From f132f3ecbc416aeb24d96ff61a674fab7bb98bfa Mon Sep 17 00:00:00 2001 From: Mason Date: Wed, 23 Jan 2019 13:20:29 -0800 Subject: [PATCH 19/21] Restored hotData to computed, but removed unnecessary side-effects --- .../components/RuleCollectionBuilder.vue | 95 +++++++++---------- .../scripts/mvc/rules/rule-definitions.js | 6 +- 2 files changed, 48 insertions(+), 53 deletions(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 2d9a8028969..05a548841fb 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -448,7 +448,7 @@ []); + columns = []; + } else if (this.elementsType == "collection_contents") { + const collection = this.initialElements.slice(); + if (collection) { + const obj = this.populateElementsFromCollectionDescription( + collection.elements, + collection.collection_type + ); + data = obj.data; + sources = obj.sources; + columns = []; + } else { + data = []; + sources = []; + columns = []; + } + } else { + data = this.initialElements.slice(); + sources = data.map(el => null); + columns = []; + if (this.initialElements) { + this.initialElements[0].forEach(() => columns.push("new")); + } + } + return RuleDefs.applyRules(data, sources, columns, this.rules); + }, + colHeadersPerRule() { + return this.hotData.colHeadersPerRule; } }, methods: { @@ -1413,42 +1449,6 @@ export default { this.rules.push(rule); } }, - hotData() { - let data, sources, columns; - if ( - this.elementsType == "datasets" || - this.elementsType == "library_datasets" || - this.elementsType == "ftp" - ) { - data = this.initialElements.map(el => []); - sources = this.initialElements.slice(); - columns = []; - } else if (this.elementsType == "collection_contents") { - const collection = this.initialElements; - if (collection) { - const obj = this.populateElementsFromCollectionDescription( - collection.elements, - collection.collection_type - ); - data = obj.data; - sources = obj.sources; - columns = []; - } else { - data = []; - sources = []; - columns = []; - } - } else { - data = this.initialElements.slice(); - sources = data.map(el => null); - columns = []; - if (this.initialElements) { - this.initialElements[0].forEach(() => columns.push("new")); - } - } - this.colHeadersPerRule = []; - return RuleDefs.applyRules(data, sources, columns, this.rules, this.colHeadersPerRule); - }, viewSource() { this.resetSource(); this.ruleView = "source"; @@ -1685,8 +1685,7 @@ export default { return identifierColumns; }, buildRequestElements(createDatasetDescription, createSubcollectionDescription, subElementProp) { - const hotData = this.hotData(); - const data = hotData["data"]; + const data = this.hotData.data; const identifierColumns = this.identifierColumns(); if (identifierColumns.length < 1) { console.log("Error but this shouldn't have happened, create button should have been disabled."); @@ -1793,9 +1792,7 @@ export default { return elementsByName; }, creationElementsFromDatasets() { - const hotData = this.hotData(); - const sources = hotData["sources"]; - const data = hotData["data"]; + const { sources, data } = this.hotData; const mappingAsDict = this.mappingAsDict; const elementsByCollectionName = this.buildRequestElements( @@ -1814,8 +1811,7 @@ export default { }, creationElementsForFetch() { // fetch elements for HDCA - const hotData = this.hotData(); - const data = hotData["data"]; + const data = this.hotData.data; const mappingAsDict = this.mappingAsDict; const elementsByCollectionName = this.buildRequestElements( @@ -1834,8 +1830,7 @@ export default { }, creationDatasetsForFetch() { // fetch elements for HDAs if not collection information specified. - const hotData = this.hotData(); - const data = hotData["data"]; + const data = this.hotData.data; const mappingAsDict = this.mappingAsDict; const datasets = []; diff --git a/client/galaxy/scripts/mvc/rules/rule-definitions.js b/client/galaxy/scripts/mvc/rules/rule-definitions.js index 4b410cb2e55..c0cb365f121 100644 --- a/client/galaxy/scripts/mvc/rules/rule-definitions.js +++ b/client/galaxy/scripts/mvc/rules/rule-definitions.js @@ -861,8 +861,8 @@ const colHeadersFor = function(data, columns) { } }; -const applyRules = function(data, sources, columns, rules, colHeadersPerRule) { - colHeadersPerRule = colHeadersPerRule || []; +const applyRules = function(data, sources, columns, rules, headersPerRule = []) { + let colHeadersPerRule = Array.from(headersPerRule); let hasRuleError = false; for (var ruleIndex in rules) { const ruleHeaders = colHeadersFor(data, columns); @@ -889,7 +889,7 @@ const applyRules = function(data, sources, columns, rules, colHeadersPerRule) { columns = res.columns || columns; } } - return { data, sources, columns }; + return { data, sources, columns, colHeadersPerRule }; }; export default { From 86eff98fc36f3897d1d201e708ddea89e376b600 Mon Sep 17 00:00:00 2001 From: Mason Date: Wed, 23 Jan 2019 14:38:45 -0800 Subject: [PATCH 20/21] Removed slice method on non-array property --- client/galaxy/scripts/components/RuleCollectionBuilder.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 05a548841fb..1a83bdbe670 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -1385,7 +1385,7 @@ export default { data = sources.map(el => []); columns = []; } else if (this.elementsType == "collection_contents") { - const collection = this.initialElements.slice(); + const collection = this.initialElements; if (collection) { const obj = this.populateElementsFromCollectionDescription( collection.elements, From 800e221c47044b02034a54727727ddadef700bef Mon Sep 17 00:00:00 2001 From: Mason Date: Wed, 23 Jan 2019 14:56:53 -0800 Subject: [PATCH 21/21] Removed commented code in RuleCollectionBuilder --- client/galaxy/scripts/components/RuleCollectionBuilder.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 1a83bdbe670..47818d25224 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -1029,7 +1029,6 @@ export default { } return { rules: rules, - // colHeadersPerRule: [], mapping: mapping, state: "build", // 'build', 'error', 'wait', ruleView: "normal", // 'normal' or 'source'