From 390643098641b64a3a8840b42c47d302e258c8cb Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 16 Feb 2021 15:37:44 +0100 Subject: [PATCH 01/70] Add tracking code to webhook for GTN stats --- config/plugins/webhooks/gtn/script.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/plugins/webhooks/gtn/script.js b/config/plugins/webhooks/gtn/script.js index 0d98fdaac03..5cf83003627 100644 --- a/config/plugins/webhooks/gtn/script.js +++ b/config/plugins/webhooks/gtn/script.js @@ -18,13 +18,13 @@ function addIframe() { fetch("/training-material/") .then((response) => { if (!response.ok) { - url = "https://training.galaxyproject.org/training-material/"; + url = "https://training.galaxyproject.org/training-material/?utm_source=webhook&utm_medium=noproxy&utm_campaign=gxy"; message = ` Click to run unavailable. `; } else { - url = "/training-material/"; + url = "/training-material/?utm_source=webhook&utm_medium=proxy&utm_campaign=gxy"; message = ""; } }) From d69a6fcd0e119238b4812abd1bc6bfa517195142 Mon Sep 17 00:00:00 2001 From: rikeshi Date: Sat, 13 Feb 2021 11:50:17 +0100 Subject: [PATCH 02/70] remove kwds arg from call to get_stored_accessible_workflow fix 'key' causing TypeError on GET request --- lib/galaxy/webapps/galaxy/api/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 214a5fe1bb9..f09e5a46ba5 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -244,7 +244,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn Lists all versions of this workflow. """ - stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id, **kwds) + stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id) return [{'version': i, 'update_time': str(w.update_time), 'steps': len(w.steps)} for i, w in enumerate(reversed(stored_workflow.workflows))] @expose_api From 9cc01de07a6478113dc8ca6ff379bce42893c55e Mon Sep 17 00:00:00 2001 From: rikeshi Date: Mon, 15 Feb 2021 17:22:30 +0000 Subject: [PATCH 03/70] Apply suggestions from code review Co-authored-by: Simon Bray <32272674+simonbray@users.noreply.github.com> --- lib/galaxy/webapps/galaxy/api/workflows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index f09e5a46ba5..2b9ef427d2d 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -244,7 +244,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn Lists all versions of this workflow. """ - stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id) + instance = util.string_as_bool(kwds.get("instance", "false")) + stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id, by_stored_id=not instance) return [{'version': i, 'update_time': str(w.update_time), 'steps': len(w.steps)} for i, w in enumerate(reversed(stored_workflow.workflows))] @expose_api From 6117d0cbe3d2a0c7b273a681ede7cdf117415346 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 17 Feb 2021 22:15:29 +0100 Subject: [PATCH 04/70] fix slurm runner for real user setups for finished jobs (state DONE) the SLURM runner modifies the job's stderr file. for real user setups this does not work because the job dir is still owned by the real user. solution just add a reclaim ownnership before writing --- lib/galaxy/jobs/runners/slurm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index 446319df3d5..ac1f6d62bb8 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -153,6 +153,7 @@ class SlurmJobRunner(DRMAAJobRunner): self.work_queue.put((self.fail_job, ajs)) return if drmaa_state == self.drmaa_job_states.DONE: + ajs.job_wrapper.reclaim_ownership() with open(ajs.error_file) as rfh: _remove_spurious_top_lines(rfh, ajs) with open(ajs.error_file, 'r+') as f: From c18af6f5cf5d4b18adeca9ba8f913025d0008997 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 17 Feb 2021 22:18:03 +0100 Subject: [PATCH 05/70] avoid unnecessary chowns in 6117d0cbe3 another reclaim_ownership was introduced. I observed that this function is now called 3x. so lets just skip this of the dir is already owned by the corresponding owner --- lib/galaxy/util/path/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/util/path/__init__.py b/lib/galaxy/util/path/__init__.py index 9022aaabb1b..a6bd071ce40 100644 --- a/lib/galaxy/util/path/__init__.py +++ b/lib/galaxy/util/path/__init__.py @@ -32,6 +32,7 @@ from os.path import ( relpath, sep as separator, ) +from pathlib import Path try: from pwd import getpwuid except ImportError: @@ -352,6 +353,9 @@ def external_chown(path, pwent, external_chown_script, description="file"): try: if not external_chown_script: raise ValueError('external_chown_script is not defined') + if Path(path).owner() == pwent[0]: + return True + cmd = shlex.split(external_chown_script) cmd.extend([path, pwent[0], str(pwent[3])]) log.debug('Changing ownership of {} with: {}'.format(path, ' '.join(map(shlex.quote, cmd)))) From e2076fbb9dd7c79913a4c696d9c1eed5ec031add Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Feb 2021 18:08:31 +0100 Subject: [PATCH 06/70] fix genration of config files --- lib/galaxy/tools/parameters/wrapped_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index b65063c7262..a0574981292 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -97,7 +97,7 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"): if input.multiple and packaging.version.parse(str(profile)) >= packaging.version.parse('20.05'): json_value = [_ for _ in _cast_if_not_none(value_wrapper.value, list)] else: - json_value = _cast_if_not_none(value_wrapper, str) + json_value = _cast_if_not_none(value_wrapper.value, str) elif input_type == "data_column": # value is a SelectToolParameterWrapper() if input.multiple: From bd5e8f326cc63efe3a2711836c10a57d88b9ca9d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Feb 2021 19:22:29 +0100 Subject: [PATCH 07/70] fix profile logic for configfile generation some follow up fixes to https://github.com/galaxyproject/galaxy/pull/9279/files https://github.com/galaxyproject/galaxy/pull/9776/files - wrong profile for the profile test tool (it used 20.01, but should have used 20.05) - the non-profile test tool used wrong assumptions. now fixed and also tested against 19.09 - in the profile version `value_wrapper.value` needs to be casted for `multiple==False` otherwise we gat `"None"` (which I do not understand 100%) --- lib/galaxy/tools/parameters/wrapped_json.py | 9 ++++++--- test/functional/tools/inputs_as_json.xml | 6 +++--- test/functional/tools/inputs_as_json_profile.xml | 8 ++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index a0574981292..9a3393ffc21 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -94,10 +94,13 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"): elif input_type == "boolean": json_value = _cast_if_not_none(value_wrapper, bool) elif input_type == "select": - if input.multiple and packaging.version.parse(str(profile)) >= packaging.version.parse('20.05'): - json_value = [_ for _ in _cast_if_not_none(value_wrapper.value, list)] + if packaging.version.parse(str(profile)) < packaging.version.parse('20.05'): + json_value = _cast_if_not_none(value_wrapper, str) else: - json_value = _cast_if_not_none(value_wrapper.value, str) + if input.multiple: + json_value = [str(_) for _ in _cast_if_not_none(value_wrapper.value, list)] + else: + json_value = _cast_if_not_none(value_wrapper.value, str) elif input_type == "data_column": # value is a SelectToolParameterWrapper() if input.multiple: diff --git a/test/functional/tools/inputs_as_json.xml b/test/functional/tools/inputs_as_json.xml index 465aba5eda2..1aff602e796 100644 --- a/test/functional/tools/inputs_as_json.xml +++ b/test/functional/tools/inputs_as_json.xml @@ -27,8 +27,8 @@ if test_case == "1": assert_equals(as_dict["inttest"], 12456) assert_equals(as_dict["floattest"], 6.789) assert_equals(as_dict["radio_select"], "a_radio") - assert_equals(as_dict["optional_select"], None) - assert_equals(as_dict["optional_multiple_select"], []) + assert_equals(as_dict["optional_select"], "None") + assert_equals(as_dict["optional_multiple_select"], "None") assert_equals(as_dict["repeat"][0]["r"], "000000") assert_equals(as_dict["repeat"][1]["r"], "FFFFFF") assert_equals(as_dict["cond"]["more_text"], "fdefault") @@ -43,7 +43,7 @@ elif test_case == "2": assert_equals(as_dict["floattest"], 1.0) assert_equals(as_dict["radio_select"], "a_radio") assert_equals(as_dict["optional_select"], "a") - assert_equals(as_dict["optional_multiple_select"], ['a', 'b']) + assert_equals(as_dict["optional_multiple_select"], 'a,b') assert_equals(as_dict["repeat"][0]["r"], "000000") assert_equals(as_dict["cond"]["cond_test"], "second") assert_equals(as_dict["cond"]["more_text"], "sdefault") diff --git a/test/functional/tools/inputs_as_json_profile.xml b/test/functional/tools/inputs_as_json_profile.xml index 9fc04164878..162a6fa7345 100644 --- a/test/functional/tools/inputs_as_json_profile.xml +++ b/test/functional/tools/inputs_as_json_profile.xml @@ -1,7 +1,7 @@ - + Date: Fri, 15 Jan 2021 19:06:56 +0000 Subject: [PATCH 08/70] Sync social-auth-core requirement in data package v4.0.2 dropped support for Python 3.5 --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- packages/data/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 762cfc3a441..27fcd08ea2c 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -82,7 +82,7 @@ pyparsing = "*" "Fabric3" = "*" paramiko = "*" cloudbridge = "*" -social_auth_core = {version = "==3.3.0", extras = ['openidconnect']} +social-auth-core = {version = "==3.3.0", extras = ['openidconnect']} cloudauthz = "==0.6.0" gxformat2 = "*" refgenconf = ">=0.7.0" diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 98424dcbef5..8753dc5cddd 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -9,7 +9,7 @@ parsley numpy<=1.16 pycryptodome pysam -social_auth_core +social-auth-core[openidconnect]==3.3.0 SQLAlchemy sqlalchemy-migrate sqlalchemy-utils From 5e129c1f6a5f97abe329d7ea42cabeba6116b349 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 17 Feb 2021 20:12:07 +0100 Subject: [PATCH 09/70] Add API tests for groups --- lib/galaxy_test/api/test_groups.py | 112 +++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 lib/galaxy_test/api/test_groups.py diff --git a/lib/galaxy_test/api/test_groups.py b/lib/galaxy_test/api/test_groups.py new file mode 100644 index 00000000000..c2c508ac48f --- /dev/null +++ b/lib/galaxy_test/api/test_groups.py @@ -0,0 +1,112 @@ +import json + +from galaxy_test.base.populators import DatasetPopulator +from ._framework import ApiTestCase + + +class GroupsApiTestCase(ApiTestCase): + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + + def test_create_valid(self, group_name: str = None): + payload = self._build_valid_group_payload(group_name) + response = self._post("groups", payload, admin=True, json=True) + self._assert_status_code_is(response, 200) + group = response.json()[0] # POST /api/groups returns a list + self._assert_valid_group(group) + return group + + def test_create_only_admin(self): + response = self._post("groups", json=True) + self._assert_status_code_is(response, 403) + + def test_create_invalid_params_raises_400(self): + payload = self._build_valid_group_payload() + payload["name"] = None + response = self._post("groups", payload, admin=True, json=True) + self._assert_status_code_is(response, 400) + + def test_create_duplicated_name_raises_400(self): + payload = self._build_valid_group_payload() + response = self._post("groups", payload, admin=True, json=True) + self._assert_status_code_is(response, 200) + + response = self._post("groups", payload, admin=True, json=True) + self._assert_status_code_is(response, 400) + + def test_index(self): + self.test_create_valid() + response = self._get("groups", admin=True) + self._assert_status_code_is(response, 200) + groups = response.json() + assert isinstance(groups, list) + assert len(groups) > 0 + for group in groups: + self._assert_valid_group(group) + + def test_index_only_admin(self): + response = self._get("groups") + self._assert_status_code_is(response, 403) + + def test_show(self): + group = self.test_create_valid() + group_id = group["id"] + response = self._get(f"groups/{group_id}", admin=True) + self._assert_status_code_is(response, 200) + response_group = response.json() + self._assert_valid_group(response_group) + self._assert_has_keys(response_group, "users_url", "roles_url") + + def test_show_only_admin(self): + group = self.test_create_valid() + group_id = group["id"] + response = self._get(f"groups/{group_id}") + self._assert_status_code_is(response, 403) + + def test_update(self): + group = self.test_create_valid(group_name="group-test") + + group_id = group["id"] + updated_name = "group-test-updated" + update_payload = json.dumps({ + "name": updated_name, + }) + update_response = self._put(f"groups/{group_id}", data=update_payload, admin=True) + self._assert_status_code_is_ok(update_response) + + def test_update_only_admin(self): + group = self.test_create_valid() + group_id = group["id"] + response = self._put(f"groups/{group_id}") + self._assert_status_code_is(response, 403) + + def test_update_duplicating_name_raises_400(self): + group_a = self.test_create_valid() + group_b = self.test_create_valid() + + # Update group_b with the same name as group_a + group_b_id = group_b["id"] + updated_name = group_a["name"] + update_payload = json.dumps({ + "name": updated_name, + }) + update_response = self._put(f"groups/{group_b_id}", data=update_payload, admin=True) + self._assert_error_code_is(update_response, 400) + + def _assert_valid_group(self, group, assert_id=None): + self._assert_has_keys(group, "id", "name", "model_class", "url") + if assert_id is not None: + assert group["id"] == assert_id + + def _build_valid_group_payload(self, name: str = None): + name = name or self.dataset_populator.get_random_name() + user_id = self.dataset_populator.user_id() + role_id = self.dataset_populator.user_private_role_id() + payload = { + "name": name, + "user_ids": [user_id], + "role_ids": [role_id], + } + return payload From 719d8139eba4042d68c40090eaee2615e9f37eb7 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 17 Feb 2021 20:48:45 +0100 Subject: [PATCH 10/70] Replace legacy expose_api decorators xref #11252 --- lib/galaxy/webapps/galaxy/api/groups.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/groups.py b/lib/galaxy/webapps/galaxy/api/groups.py index a4ff3c90dd5..1f71fda1598 100644 --- a/lib/galaxy/webapps/galaxy/api/groups.py +++ b/lib/galaxy/webapps/galaxy/api/groups.py @@ -5,7 +5,10 @@ import logging from sqlalchemy import false -from galaxy import web +from galaxy.web import ( + expose_api, + require_admin, +) from galaxy.webapps.base.controller import BaseAPIController, url_for log = logging.getLogger(__name__) @@ -13,8 +16,8 @@ log = logging.getLogger(__name__) class GroupAPIController(BaseAPIController): - @web.require_admin - @web.legacy_expose_api + @expose_api + @require_admin def index(self, trans, **kwd): """ GET /api/groups @@ -29,7 +32,8 @@ class GroupAPIController(BaseAPIController): rval.append(item) return rval - @web.legacy_expose_api + @expose_api + @require_admin def create(self, trans, payload, **kwd): """ POST /api/groups @@ -71,8 +75,8 @@ class GroupAPIController(BaseAPIController): item['url'] = url_for('group', id=encoded_id) return [item] - @web.require_admin - @web.legacy_expose_api + @expose_api + @require_admin def show(self, trans, id, **kwd): """ GET /api/groups/{encoded_group_id} @@ -97,8 +101,8 @@ class GroupAPIController(BaseAPIController): item['roles_url'] = url_for('group_roles', group_id=group_id) return item - @web.require_admin - @web.legacy_expose_api + @expose_api + @require_admin def update(self, trans, id, payload, **kwd): """ PUT /api/groups/{encoded_group_id} From 4fdf4c50ef066fd8f4727064e715f67a626d215c Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Feb 2021 12:18:24 +0100 Subject: [PATCH 11/70] Refactor move controller logic into GroupsManager --- lib/galaxy/managers/groups.py | 107 ++++++++++++++++++++++++ lib/galaxy/webapps/galaxy/api/groups.py | 97 +++------------------ 2 files changed, 117 insertions(+), 87 deletions(-) create mode 100644 lib/galaxy/managers/groups.py diff --git a/lib/galaxy/managers/groups.py b/lib/galaxy/managers/groups.py new file mode 100644 index 00000000000..e32a6dc4378 --- /dev/null +++ b/lib/galaxy/managers/groups.py @@ -0,0 +1,107 @@ +from sqlalchemy import false + +from galaxy.web import url_for + + +class GroupsManager: + """Interface/service object shared by controllers for interacting with groups.""" + + def index(self, trans, **kwd): + """ + Displays a collection (list) of groups. + """ + rval = [] + for group in trans.sa_session.query(trans.app.model.Group).filter(trans.app.model.Group.table.c.deleted == false()): + if trans.user_is_admin: + item = group.to_dict(value_mapper={'id': trans.security.encode_id}) + encoded_id = trans.security.encode_id(group.id) + item['url'] = url_for('group', id=encoded_id) + rval.append(item) + return rval + + def create(self, trans, payload, **kwd): + """ + Creates a new group. + """ + if not trans.user_is_admin: + trans.response.status = 403 + return "You are not authorized to create a new group." + name = payload.get('name', None) + if not name: + trans.response.status = 400 + return "Enter a valid name" + if trans.sa_session.query(trans.app.model.Group).filter(trans.app.model.Group.table.c.name == name).first(): + trans.response.status = 400 + return "A group with that name already exists" + + group = trans.app.model.Group(name=name) + trans.sa_session.add(group) + user_ids = payload.get('user_ids', []) + users = [trans.sa_session.query(trans.model.User).get(trans.security.decode_id(i)) for i in user_ids] + role_ids = payload.get('role_ids', []) + roles = [trans.sa_session.query(trans.model.Role).get(trans.security.decode_id(i)) for i in role_ids] + trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users) + """ + # Create the UserGroupAssociations + for user in users: + trans.app.security_agent.associate_user_group( user, group ) + # Create the GroupRoleAssociations + for role in roles: + trans.app.security_agent.associate_group_role( group, role ) + """ + trans.sa_session.flush() + encoded_id = trans.security.encode_id(group.id) + item = group.to_dict(view='element', value_mapper={'id': trans.security.encode_id}) + item['url'] = url_for('group', id=encoded_id) + return [item] + + def show(self, trans, id, **kwd): + """ + Displays information about a group. + """ + group_id = id + try: + decoded_group_id = trans.security.decode_id(group_id) + except TypeError: + trans.response.status = 400 + return "Malformed group id ( %s ) specified, unable to decode." % str(group_id) + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + except Exception: + group = None + if not group: + trans.response.status = 400 + return "Invalid group id ( %s ) specified." % str(group_id) + item = group.to_dict(view='element', value_mapper={'id': trans.security.encode_id}) + item['url'] = url_for('group', id=group_id) + item['users_url'] = url_for('group_users', group_id=group_id) + item['roles_url'] = url_for('group_roles', group_id=group_id) + return item + + def update(self, trans, id, payload, **kwd): + """ + Modifies a group. + """ + group_id = id + try: + decoded_group_id = trans.security.decode_id(group_id) + except TypeError: + trans.response.status = 400 + return "Malformed group id ( %s ) specified, unable to decode." % str(group_id) + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + except Exception: + group = None + if not group: + trans.response.status = 400 + return "Invalid group id ( %s ) specified." % str(group_id) + name = payload.get('name', None) + if name: + group.name = name + trans.sa_session.add(group) + user_ids = payload.get('user_ids', []) + users = [trans.sa_session.query(trans.model.User).get(trans.security.decode_id(i)) for i in user_ids] + role_ids = payload.get('role_ids', []) + roles = [trans.sa_session.query(trans.model.Role).get(trans.security.decode_id(i)) for i in role_ids] + trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users, delete_existing_assocs=False) + trans.sa_session.flush() diff --git a/lib/galaxy/webapps/galaxy/api/groups.py b/lib/galaxy/webapps/galaxy/api/groups.py index 1f71fda1598..5b55c26d482 100644 --- a/lib/galaxy/webapps/galaxy/api/groups.py +++ b/lib/galaxy/webapps/galaxy/api/groups.py @@ -3,19 +3,22 @@ API operations on Group objects. """ import logging -from sqlalchemy import false - +from galaxy.managers.groups import GroupsManager from galaxy.web import ( expose_api, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController, url_for +from galaxy.webapps.base.controller import BaseAPIController log = logging.getLogger(__name__) class GroupAPIController(BaseAPIController): + def __init__(self, app): + super().__init__(app) + self.manager = GroupsManager() + @expose_api @require_admin def index(self, trans, **kwd): @@ -23,14 +26,7 @@ class GroupAPIController(BaseAPIController): GET /api/groups Displays a collection (list) of groups. """ - rval = [] - for group in trans.sa_session.query(trans.app.model.Group).filter(trans.app.model.Group.table.c.deleted == false()): - if trans.user_is_admin: - item = group.to_dict(value_mapper={'id': trans.security.encode_id}) - encoded_id = trans.security.encode_id(group.id) - item['url'] = url_for('group', id=encoded_id) - rval.append(item) - return rval + return self.manager.index(trans) @expose_api @require_admin @@ -39,41 +35,7 @@ class GroupAPIController(BaseAPIController): POST /api/groups Creates a new group. """ - log.info("groups payload%s\n" % (payload)) - if not trans.user_is_admin: - trans.response.status = 403 - return "You are not authorized to create a new group." - name = payload.get('name', None) - if not name: - trans.response.status = 400 - return "Enter a valid name" - if trans.sa_session.query(trans.app.model.Group).filter(trans.app.model.Group.table.c.name == name).first(): - trans.response.status = 400 - return "A group with that name already exists" - - group = trans.app.model.Group(name=name) - trans.sa_session.add(group) - user_ids = payload.get('user_ids', []) - for i in user_ids: - log.info("user_id: %s\n" % (i)) - log.info("{} {}\n".format(i, trans.security.decode_id(i))) - users = [trans.sa_session.query(trans.model.User).get(trans.security.decode_id(i)) for i in user_ids] - role_ids = payload.get('role_ids', []) - roles = [trans.sa_session.query(trans.model.Role).get(trans.security.decode_id(i)) for i in role_ids] - trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users) - """ - # Create the UserGroupAssociations - for user in users: - trans.app.security_agent.associate_user_group( user, group ) - # Create the GroupRoleAssociations - for role in roles: - trans.app.security_agent.associate_group_role( group, role ) - """ - trans.sa_session.flush() - encoded_id = trans.security.encode_id(group.id) - item = group.to_dict(view='element', value_mapper={'id': trans.security.encode_id}) - item['url'] = url_for('group', id=encoded_id) - return [item] + return self.manager.create(trans, payload) @expose_api @require_admin @@ -82,24 +44,7 @@ class GroupAPIController(BaseAPIController): GET /api/groups/{encoded_group_id} Displays information about a group. """ - group_id = id - try: - decoded_group_id = trans.security.decode_id(group_id) - except TypeError: - trans.response.status = 400 - return "Malformed group id ( %s ) specified, unable to decode." % str(group_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - except Exception: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str(group_id) - item = group.to_dict(view='element', value_mapper={'id': trans.security.encode_id}) - item['url'] = url_for('group', id=group_id) - item['users_url'] = url_for('group_users', group_id=group_id) - item['roles_url'] = url_for('group_roles', group_id=group_id) - return item + return self.manager.show(trans, id) @expose_api @require_admin @@ -108,26 +53,4 @@ class GroupAPIController(BaseAPIController): PUT /api/groups/{encoded_group_id} Modifies a group. """ - group_id = id - try: - decoded_group_id = trans.security.decode_id(group_id) - except TypeError: - trans.response.status = 400 - return "Malformed group id ( %s ) specified, unable to decode." % str(group_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - except Exception: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str(group_id) - name = payload.get('name', None) - if name: - group.name = name - trans.sa_session.add(group) - user_ids = payload.get('user_ids', []) - users = [trans.sa_session.query(trans.model.User).get(trans.security.decode_id(i)) for i in user_ids] - role_ids = payload.get('role_ids', []) - roles = [trans.sa_session.query(trans.model.Role).get(trans.security.decode_id(i)) for i in role_ids] - trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users, delete_existing_assocs=False) - trans.sa_session.flush() + self.manager.update(trans, id, payload) From 71214ae5675fd635f37252cc33573929db7ef761 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Feb 2021 13:01:54 +0100 Subject: [PATCH 12/70] Add basic type annotations --- lib/galaxy/managers/groups.py | 34 +++++++++++++++---------- lib/galaxy/webapps/galaxy/api/groups.py | 14 +++++++--- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/managers/groups.py b/lib/galaxy/managers/groups.py index e32a6dc4378..6150b539a3f 100644 --- a/lib/galaxy/managers/groups.py +++ b/lib/galaxy/managers/groups.py @@ -1,17 +1,25 @@ +from typing import ( + Any, + Dict, +) + from sqlalchemy import false +from galaxy import model +from galaxy.managers.context import ProvidesAppContext +from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.web import url_for class GroupsManager: """Interface/service object shared by controllers for interacting with groups.""" - def index(self, trans, **kwd): + def index(self, trans: ProvidesAppContext): """ Displays a collection (list) of groups. """ rval = [] - for group in trans.sa_session.query(trans.app.model.Group).filter(trans.app.model.Group.table.c.deleted == false()): + for group in trans.sa_session.query(model.Group).filter(model.Group.table.c.deleted == false()): if trans.user_is_admin: item = group.to_dict(value_mapper={'id': trans.security.encode_id}) encoded_id = trans.security.encode_id(group.id) @@ -19,7 +27,7 @@ class GroupsManager: rval.append(item) return rval - def create(self, trans, payload, **kwd): + def create(self, trans: ProvidesAppContext, payload: Dict[str, Any]): """ Creates a new group. """ @@ -30,16 +38,16 @@ class GroupsManager: if not name: trans.response.status = 400 return "Enter a valid name" - if trans.sa_session.query(trans.app.model.Group).filter(trans.app.model.Group.table.c.name == name).first(): + if trans.sa_session.query(model.Group).filter(model.Group.table.c.name == name).first(): trans.response.status = 400 return "A group with that name already exists" - group = trans.app.model.Group(name=name) + group = model.Group(name=name) trans.sa_session.add(group) user_ids = payload.get('user_ids', []) - users = [trans.sa_session.query(trans.model.User).get(trans.security.decode_id(i)) for i in user_ids] + users = [trans.sa_session.query(model.User).get(trans.security.decode_id(i)) for i in user_ids] role_ids = payload.get('role_ids', []) - roles = [trans.sa_session.query(trans.model.Role).get(trans.security.decode_id(i)) for i in role_ids] + roles = [trans.sa_session.query(model.Role).get(trans.security.decode_id(i)) for i in role_ids] trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users) """ # Create the UserGroupAssociations @@ -55,7 +63,7 @@ class GroupsManager: item['url'] = url_for('group', id=encoded_id) return [item] - def show(self, trans, id, **kwd): + def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField): """ Displays information about a group. """ @@ -66,7 +74,7 @@ class GroupsManager: trans.response.status = 400 return "Malformed group id ( %s ) specified, unable to decode." % str(group_id) try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + group = trans.sa_session.query(model.Group).get(decoded_group_id) except Exception: group = None if not group: @@ -78,7 +86,7 @@ class GroupsManager: item['roles_url'] = url_for('group_roles', group_id=group_id) return item - def update(self, trans, id, payload, **kwd): + def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, payload: Dict[str, Any]): """ Modifies a group. """ @@ -89,7 +97,7 @@ class GroupsManager: trans.response.status = 400 return "Malformed group id ( %s ) specified, unable to decode." % str(group_id) try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + group = trans.sa_session.query(model.Group).get(decoded_group_id) except Exception: group = None if not group: @@ -100,8 +108,8 @@ class GroupsManager: group.name = name trans.sa_session.add(group) user_ids = payload.get('user_ids', []) - users = [trans.sa_session.query(trans.model.User).get(trans.security.decode_id(i)) for i in user_ids] + users = [trans.sa_session.query(model.User).get(trans.security.decode_id(i)) for i in user_ids] role_ids = payload.get('role_ids', []) - roles = [trans.sa_session.query(trans.model.Role).get(trans.security.decode_id(i)) for i in role_ids] + roles = [trans.sa_session.query(model.Role).get(trans.security.decode_id(i)) for i in role_ids] trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users, delete_existing_assocs=False) trans.sa_session.flush() diff --git a/lib/galaxy/webapps/galaxy/api/groups.py b/lib/galaxy/webapps/galaxy/api/groups.py index 5b55c26d482..43386c22b93 100644 --- a/lib/galaxy/webapps/galaxy/api/groups.py +++ b/lib/galaxy/webapps/galaxy/api/groups.py @@ -2,8 +2,14 @@ API operations on Group objects. """ import logging +from typing import ( + Any, + Dict, +) +from galaxy.managers.context import ProvidesAppContext from galaxy.managers.groups import GroupsManager +from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.web import ( expose_api, require_admin, @@ -21,7 +27,7 @@ class GroupAPIController(BaseAPIController): @expose_api @require_admin - def index(self, trans, **kwd): + def index(self, trans: ProvidesAppContext, **kwd): """ GET /api/groups Displays a collection (list) of groups. @@ -30,7 +36,7 @@ class GroupAPIController(BaseAPIController): @expose_api @require_admin - def create(self, trans, payload, **kwd): + def create(self, trans: ProvidesAppContext, payload: Dict[str, Any], **kwd): """ POST /api/groups Creates a new group. @@ -39,7 +45,7 @@ class GroupAPIController(BaseAPIController): @expose_api @require_admin - def show(self, trans, id, **kwd): + def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, **kwd): """ GET /api/groups/{encoded_group_id} Displays information about a group. @@ -48,7 +54,7 @@ class GroupAPIController(BaseAPIController): @expose_api @require_admin - def update(self, trans, id, payload, **kwd): + def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, payload: Dict[str, Any], **kwd): """ PUT /api/groups/{encoded_group_id} Modifies a group. From 559ba46302a57b8a6621bd14ea2656228bb4e886 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Feb 2021 14:33:09 +0100 Subject: [PATCH 13/70] Refactor error handling logic --- lib/galaxy/managers/groups.py | 100 ++++++++++-------------- lib/galaxy/webapps/galaxy/api/groups.py | 5 +- 2 files changed, 46 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/managers/groups.py b/lib/galaxy/managers/groups.py index 6150b539a3f..1c85418b2a0 100644 --- a/lib/galaxy/managers/groups.py +++ b/lib/galaxy/managers/groups.py @@ -6,6 +6,13 @@ from typing import ( from sqlalchemy import false from galaxy import model +from galaxy.app import StructuredApp +from galaxy.exceptions import ( + Conflict, + ObjectAttributeMissingException, + ObjectNotFound, +) +from galaxy.managers.base import decode_id from galaxy.managers.context import ProvidesAppContext from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.web import url_for @@ -14,102 +21,81 @@ from galaxy.web import url_for class GroupsManager: """Interface/service object shared by controllers for interacting with groups.""" + def __init__(self, app: StructuredApp) -> None: + self._app = app + def index(self, trans: ProvidesAppContext): """ Displays a collection (list) of groups. """ rval = [] for group in trans.sa_session.query(model.Group).filter(model.Group.table.c.deleted == false()): - if trans.user_is_admin: - item = group.to_dict(value_mapper={'id': trans.security.encode_id}) - encoded_id = trans.security.encode_id(group.id) - item['url'] = url_for('group', id=encoded_id) - rval.append(item) + item = group.to_dict(value_mapper={'id': trans.security.encode_id}) + encoded_id = trans.security.encode_id(group.id) + item['url'] = url_for('group', id=encoded_id) + rval.append(item) return rval def create(self, trans: ProvidesAppContext, payload: Dict[str, Any]): """ Creates a new group. """ - if not trans.user_is_admin: - trans.response.status = 403 - return "You are not authorized to create a new group." name = payload.get('name', None) - if not name: - trans.response.status = 400 - return "Enter a valid name" - if trans.sa_session.query(model.Group).filter(model.Group.table.c.name == name).first(): - trans.response.status = 400 - return "A group with that name already exists" + if name is None: + raise ObjectAttributeMissingException("Missing required name") + self._check_duplicated_group_name(trans, name) group = model.Group(name=name) trans.sa_session.add(group) user_ids = payload.get('user_ids', []) - users = [trans.sa_session.query(model.User).get(trans.security.decode_id(i)) for i in user_ids] + users = [trans.sa_session.query(model.User).get(self._decode_id(i)) for i in user_ids] role_ids = payload.get('role_ids', []) - roles = [trans.sa_session.query(model.Role).get(trans.security.decode_id(i)) for i in role_ids] + roles = [trans.sa_session.query(model.Role).get(self._decode_id(i)) for i in role_ids] trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users) - """ - # Create the UserGroupAssociations - for user in users: - trans.app.security_agent.associate_user_group( user, group ) - # Create the GroupRoleAssociations - for role in roles: - trans.app.security_agent.associate_group_role( group, role ) - """ trans.sa_session.flush() + encoded_id = trans.security.encode_id(group.id) item = group.to_dict(view='element', value_mapper={'id': trans.security.encode_id}) item['url'] = url_for('group', id=encoded_id) return [item] - def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField): + def show(self, trans: ProvidesAppContext, encoded_id: EncodedDatabaseIdField): """ Displays information about a group. """ - group_id = id - try: - decoded_group_id = trans.security.decode_id(group_id) - except TypeError: - trans.response.status = 400 - return "Malformed group id ( %s ) specified, unable to decode." % str(group_id) - try: - group = trans.sa_session.query(model.Group).get(decoded_group_id) - except Exception: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str(group_id) + group = self._get_group(trans, encoded_id) item = group.to_dict(view='element', value_mapper={'id': trans.security.encode_id}) - item['url'] = url_for('group', id=group_id) - item['users_url'] = url_for('group_users', group_id=group_id) - item['roles_url'] = url_for('group_roles', group_id=group_id) + item['url'] = url_for('group', id=encoded_id) + item['users_url'] = url_for('group_users', group_id=encoded_id) + item['roles_url'] = url_for('group_roles', group_id=encoded_id) return item - def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, payload: Dict[str, Any]): + def update(self, trans: ProvidesAppContext, encoded_id: EncodedDatabaseIdField, payload: Dict[str, Any]): """ Modifies a group. """ - group_id = id - try: - decoded_group_id = trans.security.decode_id(group_id) - except TypeError: - trans.response.status = 400 - return "Malformed group id ( %s ) specified, unable to decode." % str(group_id) - try: - group = trans.sa_session.query(model.Group).get(decoded_group_id) - except Exception: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str(group_id) + group = self._get_group(trans, encoded_id) name = payload.get('name', None) if name: group.name = name trans.sa_session.add(group) user_ids = payload.get('user_ids', []) - users = [trans.sa_session.query(model.User).get(trans.security.decode_id(i)) for i in user_ids] + users = [trans.sa_session.query(model.User).get(self._decode_id(i)) for i in user_ids] role_ids = payload.get('role_ids', []) - roles = [trans.sa_session.query(model.Role).get(trans.security.decode_id(i)) for i in role_ids] + roles = [trans.sa_session.query(model.Role).get(self._decode_id(i)) for i in role_ids] trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users, delete_existing_assocs=False) trans.sa_session.flush() + + def _decode_id(self, encoded_id: EncodedDatabaseIdField) -> int: + return decode_id(self._app, encoded_id) + + def _check_duplicated_group_name(self, trans: ProvidesAppContext, group_name: str) -> None: + if trans.sa_session.query(model.Group).filter(model.Group.table.c.name == group_name).first(): + raise Conflict(f"A group with name '{group_name}' already exists") + + def _get_group(self, trans: ProvidesAppContext, encoded_id: EncodedDatabaseIdField) -> model.Group: + decoded_group_id = self._decode_id(encoded_id) + group = trans.sa_session.query(model.Group).get(decoded_group_id) + if group is None: + raise ObjectNotFound(f"Group with id {encoded_id} was not found.") + return group diff --git a/lib/galaxy/webapps/galaxy/api/groups.py b/lib/galaxy/webapps/galaxy/api/groups.py index 43386c22b93..9433dab6ff6 100644 --- a/lib/galaxy/webapps/galaxy/api/groups.py +++ b/lib/galaxy/webapps/galaxy/api/groups.py @@ -7,6 +7,7 @@ from typing import ( Dict, ) +from galaxy.app import StructuredApp from galaxy.managers.context import ProvidesAppContext from galaxy.managers.groups import GroupsManager from galaxy.schema.fields import EncodedDatabaseIdField @@ -21,9 +22,9 @@ log = logging.getLogger(__name__) class GroupAPIController(BaseAPIController): - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) - self.manager = GroupsManager() + self.manager = GroupsManager(app) @expose_api @require_admin From d38ba53b212f8d24d76e1d9523e5eb9dc3e90a62 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Feb 2021 14:45:31 +0100 Subject: [PATCH 14/70] Fix groups update endpoint Prevent possible name duplication and add error handling. Fix tests to assert 409 when the operation may result in a name conflict. --- lib/galaxy/managers/groups.py | 1 + lib/galaxy_test/api/test_groups.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/managers/groups.py b/lib/galaxy/managers/groups.py index 1c85418b2a0..0e43b91336b 100644 --- a/lib/galaxy/managers/groups.py +++ b/lib/galaxy/managers/groups.py @@ -77,6 +77,7 @@ class GroupsManager: group = self._get_group(trans, encoded_id) name = payload.get('name', None) if name: + self._check_duplicated_group_name(trans, name) group.name = name trans.sa_session.add(group) user_ids = payload.get('user_ids', []) diff --git a/lib/galaxy_test/api/test_groups.py b/lib/galaxy_test/api/test_groups.py index c2c508ac48f..66606be3dc6 100644 --- a/lib/galaxy_test/api/test_groups.py +++ b/lib/galaxy_test/api/test_groups.py @@ -28,13 +28,13 @@ class GroupsApiTestCase(ApiTestCase): response = self._post("groups", payload, admin=True, json=True) self._assert_status_code_is(response, 400) - def test_create_duplicated_name_raises_400(self): + def test_create_duplicated_name_raises_409(self): payload = self._build_valid_group_payload() response = self._post("groups", payload, admin=True, json=True) self._assert_status_code_is(response, 200) response = self._post("groups", payload, admin=True, json=True) - self._assert_status_code_is(response, 400) + self._assert_status_code_is(response, 409) def test_index(self): self.test_create_valid() @@ -65,6 +65,11 @@ class GroupsApiTestCase(ApiTestCase): response = self._get(f"groups/{group_id}") self._assert_status_code_is(response, 403) + def test_show_unknown_raises_400(self): + group_id = "invalid-group-id" + response = self._get(f"groups/{group_id}", admin=True) + self._assert_status_code_is(response, 400) + def test_update(self): group = self.test_create_valid(group_name="group-test") @@ -82,7 +87,7 @@ class GroupsApiTestCase(ApiTestCase): response = self._put(f"groups/{group_id}") self._assert_status_code_is(response, 403) - def test_update_duplicating_name_raises_400(self): + def test_update_duplicating_name_raises_409(self): group_a = self.test_create_valid() group_b = self.test_create_valid() @@ -93,7 +98,7 @@ class GroupsApiTestCase(ApiTestCase): "name": updated_name, }) update_response = self._put(f"groups/{group_b_id}", data=update_payload, admin=True) - self._assert_error_code_is(update_response, 400) + self._assert_status_code_is(update_response, 409) def _assert_valid_group(self, group, assert_id=None): self._assert_has_keys(group, "id", "name", "model_class", "url") From fc1d74d1d3a7d8c9d5a07dd71a14ab8caaec1474 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 8 Feb 2021 11:19:41 +0100 Subject: [PATCH 15/70] Deploy postgresql and rabbitmq in kubernetes Setting up minikube stops the postgres and rabbitmq containers that are set up by the workflow. These are setup with custom docker networks, so it isn't trivial to restart them. Instead we just deploy them to minikube. --- .ci/minikube-test-setup/deployment.yaml | 32 +++++++++ .ci/minikube-test-setup/start_services.sh | 12 ++++ .github/workflows/integration.yaml | 85 +++++++++++++---------- 3 files changed, 92 insertions(+), 37 deletions(-) create mode 100644 .ci/minikube-test-setup/deployment.yaml create mode 100644 .ci/minikube-test-setup/start_services.sh diff --git a/.ci/minikube-test-setup/deployment.yaml b/.ci/minikube-test-setup/deployment.yaml new file mode 100644 index 00000000000..d7afe110c83 --- /dev/null +++ b/.ci/minikube-test-setup/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: testing + name: testing +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: test + template: + metadata: + labels: + app.kubernetes.io/name: test + spec: + containers: + - image: postgres:12 + name: postgres + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: postgres + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + value: postgres + - image: rabbitmq + name: rabbitmq + ports: + - containerPort: 5672 diff --git a/.ci/minikube-test-setup/start_services.sh b/.ci/minikube-test-setup/start_services.sh new file mode 100644 index 00000000000..1e860dadcb7 --- /dev/null +++ b/.ci/minikube-test-setup/start_services.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -ex + +SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") +kubectl apply -f "$SCRIPTDIR/deployment.yaml" +kubectl expose deployment testing --type=LoadBalancer --name=testing-service + +CLUSTER_IP=$(kubectl get service testing-service -o jsonpath='{.spec.clusterIP}') +GALAXY_TEST_DBURI="postgresql://postgres:postgres@${CLUSTER_IP}:5432/galaxy?client_encoding=utf-8" +GALAXY_TEST_AMQP_URL="amqp://${CLUSTER_IP}:5672)//" +export GALAXY_TEST_DBURI +export GALAXY_TEST_AMQP_URL diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 3bd0aaf4d69..c48e5346388 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -6,14 +6,15 @@ env: jobs: test: name: Test - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: [3.7] - subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)'] + python-version: ['3.7'] + subset: ['kubernetes'] services: postgres: - image: postgres:11 + image: postgres:13 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -25,36 +26,46 @@ jobs: ports: - 5672:5672 steps: - - name: Prune unused docker image, volumes and containers - run: docker system prune -a -f - - name: Clean dotnet folder for space - if: matrix.subset == 'kubernetes' - run: rm -Rf /usr/share/dotnet - - name: Setup Minikube - if: matrix.subset == 'kubernetes' - id: minikube - uses: CodingNagger/minikube-setup-action@v1.0.3 - with: - minikube-version: "1.9.0-0_amd64" - - name: Launch Minikube - if: matrix.subset == 'kubernetes' - run: eval ${{ steps.minikube.outputs.launcher }} - - name: Check pods - if: matrix.subset == 'kubernetes' - run: | - kubectl get pods - - uses: actions/checkout@v2 - with: - path: 'galaxy root' - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip dir - uses: actions/cache@v1 - id: pip-cache - with: - path: ~/.cache/pip - key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} - - name: Run tests - run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' - working-directory: 'galaxy root' + - name: Prune unused docker image, volumes and containers + run: docker system prune -a -f + - name: Clean dotnet folder for space + if: matrix.subset == 'kubernetes' + run: rm -Rf /usr/share/dotnet + - name: Setup Minikube + if: matrix.subset == 'kubernetes' + id: minikube + uses: CodingNagger/minikube-setup-action@v1.0.3 + with: + minikube-version: "1.9.0-0_amd64" + - name: Launch Minikube + if: matrix.subset == 'kubernetes' + run: eval ${{ steps.minikube.outputs.launcher }} + - name: Check pods + if: matrix.subset == 'kubernetes' + run: | + kubectl get pods + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip dir + uses: actions/cache@v1 + id: pip-cache + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Run tests + if: matrix.subset != 'kubernetes' + run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + working-directory: 'galaxy root' + - name: Run tests + if: matrix.subset == 'kubernetes' + run: 'source .ci/minikube-test-setup/start_services.sh && ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + working-directory: 'galaxy root' + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: Integration test results (${{ matrix.python-version }}, ${{ matrix.subset }}) + path: 'galaxy root/run_integration_tests.html' From a8adab100ccf38cfc2a14de3c798479f5531213e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 8 Feb 2021 13:08:45 +0100 Subject: [PATCH 16/70] Restore all integration subsets --- .github/workflows/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index c48e5346388..410af8045a9 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: python-version: ['3.7'] - subset: ['kubernetes'] + subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)'] services: postgres: image: postgres:13 From db1618fc6a3c7454ab2d1ed8af8ea6d38aeb66e7 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Mon, 8 Feb 2021 14:01:28 +0100 Subject: [PATCH 17/70] Use multiline yaml Co-authored-by: Nicola Soranzo --- .github/workflows/integration.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 410af8045a9..5cba3dba72b 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -62,7 +62,9 @@ jobs: working-directory: 'galaxy root' - name: Run tests if: matrix.subset == 'kubernetes' - run: 'source .ci/minikube-test-setup/start_services.sh && ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + run: | + . .ci/minikube-test-setup/start_services.sh + ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}" working-directory: 'galaxy root' - uses: actions/upload-artifact@v2 if: failure() From b0fcf86a0fd3578f1b6da2c245453ffaed508db3 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Feb 2021 16:02:27 +0100 Subject: [PATCH 18/70] Decode all ids and fetch them all in one query As @mvdbeek suggested --- lib/galaxy/managers/groups.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/managers/groups.py b/lib/galaxy/managers/groups.py index 0e43b91336b..8162bf3ebfe 100644 --- a/lib/galaxy/managers/groups.py +++ b/lib/galaxy/managers/groups.py @@ -1,6 +1,7 @@ from typing import ( Any, Dict, + List, ) from sqlalchemy import false @@ -47,10 +48,10 @@ class GroupsManager: group = model.Group(name=name) trans.sa_session.add(group) - user_ids = payload.get('user_ids', []) - users = [trans.sa_session.query(model.User).get(self._decode_id(i)) for i in user_ids] - role_ids = payload.get('role_ids', []) - roles = [trans.sa_session.query(model.Role).get(self._decode_id(i)) for i in role_ids] + encoded_user_ids = payload.get('user_ids', []) + users = self._get_users_by_encoded_ids(trans, encoded_user_ids) + encoded_role_ids = payload.get('role_ids', []) + roles = self._get_roles_by_encoded_ids(trans, encoded_role_ids) trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users) trans.sa_session.flush() @@ -80,16 +81,19 @@ class GroupsManager: self._check_duplicated_group_name(trans, name) group.name = name trans.sa_session.add(group) - user_ids = payload.get('user_ids', []) - users = [trans.sa_session.query(model.User).get(self._decode_id(i)) for i in user_ids] - role_ids = payload.get('role_ids', []) - roles = [trans.sa_session.query(model.Role).get(self._decode_id(i)) for i in role_ids] + encoded_user_ids = payload.get('user_ids', []) + users = self._get_users_by_encoded_ids(trans, encoded_user_ids) + encoded_role_ids = payload.get('role_ids', []) + roles = self._get_roles_by_encoded_ids(trans, encoded_role_ids) trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users, delete_existing_assocs=False) trans.sa_session.flush() def _decode_id(self, encoded_id: EncodedDatabaseIdField) -> int: return decode_id(self._app, encoded_id) + def _decode_ids(self, encoded_ids: List[EncodedDatabaseIdField]) -> List[int]: + return [self._decode_id(encoded_id) for encoded_id in encoded_ids] + def _check_duplicated_group_name(self, trans: ProvidesAppContext, group_name: str) -> None: if trans.sa_session.query(model.Group).filter(model.Group.table.c.name == group_name).first(): raise Conflict(f"A group with name '{group_name}' already exists") @@ -100,3 +104,13 @@ class GroupsManager: if group is None: raise ObjectNotFound(f"Group with id {encoded_id} was not found.") return group + + def _get_users_by_encoded_ids(self, trans: ProvidesAppContext, encoded_user_ids: List[EncodedDatabaseIdField]) -> List[model.User]: + decoded_user_ids = self._decode_ids(encoded_user_ids) + users = trans.sa_session.query(model.User).filter(model.User.table.c.id.in_(decoded_user_ids)).all() + return users + + def _get_roles_by_encoded_ids(self, trans: ProvidesAppContext, encoded_role_ids: List[EncodedDatabaseIdField]) -> List[model.Role]: + decoded_role_ids = self._decode_ids(encoded_role_ids) + roles = trans.sa_session.query(model.Role).filter(model.Role.table.c.id.in_(decoded_role_ids)).all() + return roles From 2de1676d0a1cf226cb435bea5571c447a85f05b7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 18 Feb 2021 13:32:56 +0100 Subject: [PATCH 19/70] Use request id middleware to scope session We still need to remove the session at the end of the request (https://docs.sqlalchemy.org/en/13/orm/contextual.html#using-thread-local-scope-with-web-applications), so maybe a middleware that does this is the right thing. --- lib/galaxy/dependencies/dev-requirements.txt | 1 + .../dependencies/pinned-requirements.txt | 1 + lib/galaxy/model/base.py | 33 ++++++++----------- lib/galaxy/model/database_heartbeat.py | 2 +- lib/galaxy/webapps/galaxy/api/__init__.py | 10 ++---- lib/galaxy/webapps/galaxy/fast_app.py | 3 ++ pyproject.toml | 1 + 7 files changed, 23 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 6247364dab5..710dd4864f1 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -208,6 +208,7 @@ sqlalchemy==1.3.22; (python_version >= "2.7" and python_full_version < "3.0.0") sqlitedict==1.7.0 sqlparse==0.4.1; python_version >= "3.5" starlette==0.13.6; python_version >= "3.6" and python_version < "4.0" +starlette-context==0.3.1; python_version >= "3.7" stevedore==3.3.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 22cb517c5d2..cb92da6600d 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -168,6 +168,7 @@ sqlalchemy==1.3.22; (python_version >= "2.7" and python_full_version < "3.0.0") sqlitedict==1.7.0 sqlparse==0.4.1; python_version >= "3.5" starlette==0.13.6; python_version >= "3.6" and python_version < "4.0" +starlette-context==0.3.1; python_version >= "3.7" stevedore==3.3.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 diff --git a/lib/galaxy/model/base.py b/lib/galaxy/model/base.py index aaf92d53289..3af9af88ebe 100644 --- a/lib/galaxy/model/base.py +++ b/lib/galaxy/model/base.py @@ -2,6 +2,7 @@ Shared model and mapping code between Galaxy and Tool Shed, trying to generalize to generic database connections. """ +import threading from inspect import ( getmembers, isclass @@ -12,10 +13,21 @@ from sqlalchemy.orm import ( scoped_session, sessionmaker ) +try: + from starlette_context import context as request_context +except ImportError: + request_context = None from galaxy.util.bunch import Bunch +def get_current_request(): + try: + return request_context.data['X-Request-ID'] + except RuntimeError: + return threading.get_ident() + + # TODO: Refactor this to be a proper class, not a bunch. class ModelMapping(Bunch): @@ -23,12 +35,12 @@ class ModelMapping(Bunch): self.engine = engine SessionLocal = sessionmaker(autoflush=False, autocommit=True) versioned_session(SessionLocal) - context = scoped_session(SessionLocal) + context = scoped_session(SessionLocal, scopefunc=get_current_request if request_context is not None else None) # For backward compatibility with "context.current" # deprecated? context.current = context self._SessionLocal = SessionLocal - self._session = context + self.session = context self.local_session = None model_classes = {} @@ -42,23 +54,6 @@ class ModelMapping(Bunch): context.remove() context.configure(bind=engine) - def set_local_session(self): - self.session = self._SessionLocal() - - def dispose_local_session(self): - self.session = None - - @property - def session(self): - return self.local_session or self._session - - @session.setter - def session(self, session): - # For backward compatibility with "context.current" - if session: - session.current = session - self.local_session = session - @property def context(self): return self.session diff --git a/lib/galaxy/model/database_heartbeat.py b/lib/galaxy/model/database_heartbeat.py index 79ca0e7abb8..982f3eeac01 100644 --- a/lib/galaxy/model/database_heartbeat.py +++ b/lib/galaxy/model/database_heartbeat.py @@ -25,7 +25,7 @@ class DatabaseHeartbeat: @property def sa_session(self): - return self.application_stack.app.model.context + return self.application_stack.app.model.session @property def server_name(self): diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 8da18edf0f6..479a316f136 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -3,7 +3,6 @@ This module *does not* contain API routes. It exclusively contains dependencies """ from typing import ( cast, - Generator, Optional, ) @@ -35,13 +34,8 @@ from galaxy.web.framework.decorators import require_admin_message from galaxy.work.context import SessionRequestContext -def get_app() -> Generator[UniverseApplication, None, None]: - app = cast(UniverseApplication, galaxy_app.app) - try: - app.model.set_local_session() - yield app - finally: - app.model.dispose_local_session() +def get_app() -> UniverseApplication: + return cast(UniverseApplication, galaxy_app.app) def get_id_encoding_helper(app: UniverseApplication = Depends(get_app)) -> IdEncodingHelper: diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index d46204354d1..b85c21bea56 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -4,6 +4,8 @@ from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.responses import JSONResponse from starlette.middleware.cors import CORSMiddleware from starlette.responses import Response +from starlette_context import plugins +from starlette_context.middleware import RawContextMiddleware from galaxy.exceptions import MessageException from galaxy.web.framework.base import walk_controller_modules @@ -116,6 +118,7 @@ def initialize_fast_app(gx_webapp, gx_app): app = FastAPI( openapi_tags=api_tags_metadata ) + app.add_middleware(RawContextMiddleware, plugins=(plugins.RequestIdPlugin(force_new_uuid=True),)) add_exception_handler(app) add_galaxy_middleware(app, gx_app) wsgi_handler = WSGIMiddleware(gx_webapp) diff --git a/pyproject.toml b/pyproject.toml index 9a497ec3db0..e229236d391 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ sqlalchemy-migrate = "*" SQLAlchemy-Utils = "!=0.36.8" # https://github.com/kvesteri/sqlalchemy-utils/issues/462 sqlitedict = "*" sqlparse = "*" +starlette-context = "*" svgwrite = "*" uvicorn = "*" WebOb = "*" From bedb80aec6dc30b9a57a1d3b846c839769a18910 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 18 Feb 2021 19:16:10 +0100 Subject: [PATCH 20/70] add UpgradeWorkflowAction --- lib/galaxy/workflow/refactor/execute.py | 207 +++++++++++++----------- lib/galaxy/workflow/refactor/schema.py | 5 + 2 files changed, 117 insertions(+), 95 deletions(-) diff --git a/lib/galaxy/workflow/refactor/execute.py b/lib/galaxy/workflow/refactor/execute.py index 89ee398efb1..8b0f7c1bd61 100644 --- a/lib/galaxy/workflow/refactor/execute.py +++ b/lib/galaxy/workflow/refactor/execute.py @@ -40,6 +40,7 @@ from .schema import ( UpdateStepPositionAction, UpgradeSubworkflowAction, UpgradeToolAction, + UpgradeWorkflowAction, ) from ..modules import ( InputParameterModule, @@ -366,107 +367,16 @@ class WorkflowRefactorExecutor: stored_workflow = old_workflow.stored_workflow content_id = trans.security.encode_id(stored_workflow.latest_workflow.id) step_def["content_id"] = content_id - upgrade_order_index = step_def["id"] - upgrade_label = step_def.get("label") - step = self.workflow.steps[upgrade_order_index] + step = self.workflow.steps[step_def["id"]] new_workflow = trans.app.workflow_manager.get_owned_workflow( trans, content_id ) step.subworkflow = new_workflow self._inject_for_updated_step(step, execution) - # TODO: rework this logic to work with tool upgrade also. - # TODO: find workflow outputs that need to be dropped and report them - upgrade_inputs = step.module.get_all_inputs() - upgrade_outputs = step.module.get_all_outputs() - upgrade_output_names = [u["name"] for u in upgrade_outputs] - all_input_connections = step_def.get("input_connections") - inputs_to_delete = [] - for input_name, input_connections in all_input_connections.items(): - matching_input = None - for upgrade_input in upgrade_inputs: - if upgrade_input["label"] == input_name: - matching_input = upgrade_input + self._patch_step(execution, step, step_def) - # In the future check parameter type, format, mapping status... - if matching_input is None: - inputs_to_delete.append(input_name) - for input_connection in _listify_connections(input_connections): - message_text = f"Subworkflow input '{input_name}' no longer available, dropping connection to it." - from_order_index = input_connection["id"] - from_step_label = self._as_dict["steps"].get(from_order_index, {}).get("label") - message = RefactorActionExecutionMessage( - message=message_text, - message_type=RefactorActionExecutionMessageTypeEnum.connection_drop_forced, - input_name=input_name, - step_label=upgrade_label, - order_index=upgrade_order_index, - output_name=input_connection["output_name"], - from_order_index=from_order_index, - from_step_label=from_step_label, - ) - execution.messages.append( - message - ) - - for input_name in inputs_to_delete: - del all_input_connections[input_name] - - for _, step in self._as_dict["steps"].items(): - all_input_connections = step.get("input_connections") - for input_name, input_connections in all_input_connections.items(): - rebuilt_valid_connections = [] - for input_connection in _listify_connections(input_connections): - include = False - if input_connection["id"] != upgrade_order_index: - include = True - else: - output_name = input_connection["output_name"] - if output_name in upgrade_output_names: - include = True - else: - # dropped outputs - message_text = f"Subworkflow output '{output_name}' no longer available, dropping connection from it." - message = RefactorActionExecutionMessage( - message=message_text, - message_type=RefactorActionExecutionMessageTypeEnum.connection_drop_forced, - input_name=input_name, - step_label=step.get("label"), - order_index=step["id"], - output_name=output_name, - from_step_label=upgrade_label, - from_order_index=upgrade_order_index, - ) - execution.messages.append( - message - ) - if include: - rebuilt_valid_connections.append(input_connection) - all_input_connections[input_name] = rebuilt_valid_connections - - workflow_outputs_to_delete = [] - for workflow_output in step_def.get("workflow_outputs", []): - output_label = workflow_output.get("label") - output_name = workflow_output["output_name"] - if not output_label: - continue - - if output_name not in upgrade_output_names: - workflow_outputs_to_delete.append(workflow_output) - message_text = f"Subworkflow output '{output_name}' no longer available, dropping corresponding workflow output label {output_label}." - message = RefactorActionExecutionMessage( - message=message_text, - message_type=RefactorActionExecutionMessageTypeEnum.workflow_output_drop_forced, - step_label=upgrade_label, - order_index=upgrade_order_index, - output_name=workflow_output.get("output_name"), - output_label=output_label, - ) - execution.messages.append( - message - ) - - def _apply_upgrade_tool(self, action: UpgradeToolAction, execution: RefactorActionExecutionMessage): + def _apply_upgrade_tool(self, action: UpgradeToolAction, execution: RefactorActionExecution): step_def = self._find_step(action.step) tool_id = step_def["content_id"] trans = self.module_injector.trans @@ -481,7 +391,16 @@ class WorkflowRefactorExecutor: self._inject_for_updated_step(step, execution) step_def["tool_version"] = tool_version step_def["tool_state"] = step.module.get_tool_state() - # TODO: recalculate connections and record messages... + self._patch_step(execution, step, step_def) + + def _apply_upgrade_workflow(self, action: UpgradeWorkflowAction, execution: RefactorActionExecution): + for step_order_index, step in self._as_dict["steps"].items(): + if step.get('type') == 'subworkflow': + action = UpgradeSubworkflowAction(action_type='upgrade_subworkflow', step={'order_index': step_order_index}) + self._apply_upgrade_subworkflow(action, execution) + elif step.get('type') == 'tool': + action = UpgradeToolAction(action_type='upgrade_tool', step={'order_index': step_order_index}) + self._apply_upgrade_tool(action, execution) def _find_step(self, step_reference: step_reference_union): order_index = None @@ -586,6 +505,104 @@ class WorkflowRefactorExecutor: output_name = output_reference.output_name return input_step_dict, input_name, output_step_dict, output_name + def _patch_step(self, execution, step, step_def): + """ + patch a workflow step after upgrading the tool / subworkflow + """ + # TODO: find workflow outputs that need to be dropped and report them + upgrade_inputs = step.module.get_all_inputs() + upgrade_outputs = step.module.get_all_outputs() + upgrade_output_names = [u["name"] for u in upgrade_outputs] + upgrade_order_index = step_def["id"] + upgrade_label = step_def.get("label") + all_input_connections = step_def.get("input_connections") + inputs_to_delete = [] + for input_name, input_connections in all_input_connections.items(): + # try and find an input connection for each input + matching_input = None + for upgrade_input in upgrade_inputs: + if upgrade_input["name"] == input_name: + matching_input = upgrade_input + break + + # In the future check parameter type, format, mapping status... + if matching_input is None: + inputs_to_delete.append(input_name) + for input_connection in _listify_connections(input_connections): + message_text = f"Tool or subworkflow input '{input_name}' no longer available, dropping connection to it." + from_order_index = input_connection["id"] + from_step_label = self._as_dict["steps"].get(from_order_index, {}).get("label") + message = RefactorActionExecutionMessage( + message=message_text, + message_type=RefactorActionExecutionMessageTypeEnum.connection_drop_forced, + input_name=input_name, + step_label=upgrade_label, + order_index=upgrade_order_index, + output_name=input_connection["output_name"], + from_order_index=from_order_index, + from_step_label=from_step_label, + ) + execution.messages.append( + message + ) + + for input_name in inputs_to_delete: + del all_input_connections[input_name] + + for _, step in self._as_dict["steps"].items(): + all_input_connections = step.get("input_connections") + for input_name, input_connections in all_input_connections.items(): + rebuilt_valid_connections = [] + for input_connection in _listify_connections(input_connections): + include = False + if input_connection["id"] != upgrade_order_index: + include = True + else: + output_name = input_connection["output_name"] + if output_name in upgrade_output_names: + include = True + else: + # dropped outputs + message_text = f"Tool or subworkflow output '{output_name}' no longer available, dropping connection from it." + message = RefactorActionExecutionMessage( + message=message_text, + message_type=RefactorActionExecutionMessageTypeEnum.connection_drop_forced, + input_name=input_name, + step_label=step.get("label"), + order_index=step["id"], + output_name=output_name, + from_step_label=upgrade_label, + from_order_index=upgrade_order_index, + ) + execution.messages.append( + message + ) + if include: + rebuilt_valid_connections.append(input_connection) + all_input_connections[input_name] = rebuilt_valid_connections + + workflow_outputs_to_delete = [] + for workflow_output in step_def.get("workflow_outputs", []): + output_label = workflow_output.get("label") + output_name = workflow_output["output_name"] + if not output_label: + continue + + if output_name not in upgrade_output_names: + workflow_outputs_to_delete.append(workflow_output) + message_text = f"Subworkflow output '{output_name}' no longer available, dropping corresponding workflow output label {output_label}." + message = RefactorActionExecutionMessage( + message=message_text, + message_type=RefactorActionExecutionMessageTypeEnum.workflow_output_drop_forced, + step_label=upgrade_label, + order_index=upgrade_order_index, + output_name=workflow_output.get("output_name"), + output_label=output_label, + ) + execution.messages.append( + message + ) + @staticmethod def normalize_input_connections_to_list(all_input_connections, input_name, add_if_missing=False): if add_if_missing and input_name not in all_input_connections: diff --git a/lib/galaxy/workflow/refactor/schema.py b/lib/galaxy/workflow/refactor/schema.py index 515c399791f..37eeda5c6c0 100644 --- a/lib/galaxy/workflow/refactor/schema.py +++ b/lib/galaxy/workflow/refactor/schema.py @@ -211,6 +211,10 @@ class UpgradeToolAction(BaseAction): tool_version: Optional[str] +class UpgradeWorkflowAction(BaseAction): + action_type: Literal['upgrade_workflow'] + + union_action_classes = Union[ AddInputAction, AddStepAction, @@ -230,6 +234,7 @@ union_action_classes = Union[ UpdateStepPositionAction, UpgradeSubworkflowAction, UpgradeToolAction, + UpgradeWorkflowAction, RemoveUnlabeledWorkflowOutputs, ] From 10ae171e52deff9214e6b9735453279cd24ed1ab Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Feb 2021 19:50:05 +0100 Subject: [PATCH 21/70] Add tests for group_roles API --- lib/galaxy_test/api/test_group_roles.py | 135 ++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 lib/galaxy_test/api/test_group_roles.py diff --git a/lib/galaxy_test/api/test_group_roles.py b/lib/galaxy_test/api/test_group_roles.py new file mode 100644 index 00000000000..f0c5450fc36 --- /dev/null +++ b/lib/galaxy_test/api/test_group_roles.py @@ -0,0 +1,135 @@ +from typing import List + +from galaxy_test.base.populators import DatasetPopulator +from ._framework import ApiTestCase + + +class GroupRolesApiTestCase(ApiTestCase): + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + + def test_index(self, group_name: str = None): + group_name = group_name or "test-group" + group = self._create_group(group_name) + encoded_group_id = group["id"] + + group_roles = self._get_group_roles(encoded_group_id) + + assert isinstance(group_roles, list) + assert len(group_roles) > 0 + for group_role in group_roles: + self._assert_valid_group_role(group_role) + + def test_index_only_admin(self): + encoded_group_id = "any-group-id" + response = self._get(f"groups/{encoded_group_id}/roles") + self._assert_status_code_is(response, 403) + + def test_index_unknown_group_raises_400(self): + encoded_group_id = "unknown-group-id" + response = self._get(f"groups/{encoded_group_id}/roles", admin=True) + self._assert_status_code_is(response, 400) + + def test_show(self): + encoded_role_id = self.dataset_populator.user_private_role_id() + group = self._create_group("test-group-show-role", encoded_role_ids=[encoded_role_id]) + encoded_group_id = group["id"] + response = self._get(f"groups/{encoded_group_id}/roles/{encoded_role_id}", admin=True) + self._assert_status_code_is(response, 200) + group_role = response.json() + self._assert_valid_group_role(group_role) + + def test_show_only_admin(self): + encoded_group_id = "any-group-id" + encoded_role_id = "any-role-id" + response = self._get(f"groups/{encoded_group_id}/roles/{encoded_role_id}") + self._assert_status_code_is(response, 403) + + def test_show_unknown_raises_400(self): + group = self._create_group("test-group-with-unknown-role") + encoded_group_id = group["id"] + encoded_role_id = "unknown-role-id" + response = self._get(f"groups/{encoded_group_id}/roles/{encoded_role_id}", admin=True) + self._assert_status_code_is(response, 400) + + def test_update(self): + group_name = "group-without-roles" + group = self._create_group(group_name, encoded_role_ids=[]) + encoded_group_id = group["id"] + + group_roles = self._get_group_roles(encoded_group_id) + assert len(group_roles) == 0 + + encoded_role_id = self.dataset_populator.user_private_role_id() + update_response = self._put(f"groups/{encoded_group_id}/roles/{encoded_role_id}", admin=True) + self._assert_status_code_is_ok(update_response) + group_role = update_response.json() + self._assert_valid_group_role(group_role, assert_id=encoded_role_id) + assert group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}" + + def test_update_only_admin(self): + encoded_group_id = "any-group-id" + encoded_role_id = "any-role-id" + response = self._put(f"groups/{encoded_group_id}/roles/{encoded_role_id}") + self._assert_status_code_is(response, 403) + + def test_delete(self): + group_name = "group-with-role-to-delete" + encoded_role_id = self.dataset_populator.user_private_role_id() + group = self._create_group(group_name, encoded_role_ids=[encoded_role_id]) + encoded_group_id = group["id"] + + group_roles = self._get_group_roles(encoded_group_id) + assert len(group_roles) == 1 + + delete_response = self._delete(f"groups/{encoded_group_id}/roles/{encoded_role_id}", admin=True) + self._assert_status_code_is_ok(delete_response) + group_role = delete_response.json() + self._assert_valid_group_role(group_role, assert_id=encoded_role_id) + + group_roles = self._get_group_roles(encoded_group_id) + assert len(group_roles) == 0 + + def test_delete_only_admin(self): + encoded_group_id = "any-group-id" + encoded_role_id = "any-role-id" + response = self._delete(f"groups/{encoded_group_id}/roles/{encoded_role_id}") + self._assert_status_code_is(response, 403) + + def test_delete_unknown_raises_400(self): + group_name = "group-without-role-to-delete" + group = self._create_group(group_name, encoded_role_ids=[]) + encoded_group_id = group["id"] + + group_roles = self._get_group_roles(encoded_group_id) + assert len(group_roles) == 0 + + encoded_role_id = "unknown-role-id" + delete_response = self._delete(f"groups/{encoded_group_id}/roles/{encoded_role_id}", admin=True) + self._assert_status_code_is(delete_response, 400) + + def _create_group(self, group_name: str, encoded_role_ids: List[str] = None): + if encoded_role_ids is None: + encoded_role_ids = [self.dataset_populator.user_private_role_id()] + role_ids = encoded_role_ids + payload = { + "name": group_name, + "role_ids": role_ids, + } + response = self._post("groups", payload, admin=True, json=True) + self._assert_status_code_is(response, 200) + group = response.json()[0] # POST /api/groups returns a list + return group + + def _get_group_roles(self, encoded_group_id: str): + response = self._get(f"groups/{encoded_group_id}/roles", admin=True) + self._assert_status_code_is(response, 200) + group_roles = response.json() + return group_roles + + def _assert_valid_group_role(self, role, assert_id=None): + self._assert_has_keys(role, "id", "name", "url") + if assert_id is not None: + assert role["id"] == assert_id From 5b718f8c9b13ef99b6872dd0a10ac914f059d0f7 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Feb 2021 19:57:04 +0100 Subject: [PATCH 22/70] Replace the old legacy_expose_api decorators --- lib/galaxy/webapps/galaxy/api/group_roles.py | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py index 8201afdf645..3f944803dd6 100644 --- a/lib/galaxy/webapps/galaxy/api/group_roles.py +++ b/lib/galaxy/webapps/galaxy/api/group_roles.py @@ -3,8 +3,11 @@ API operations on Group objects. """ import logging -from galaxy import web from galaxy.util import unicodify +from galaxy.web import ( + expose_api, + require_admin, +) from galaxy.webapps.base.controller import BaseAPIController, url_for log = logging.getLogger(__name__) @@ -12,8 +15,8 @@ log = logging.getLogger(__name__) class GroupRolesAPIController(BaseAPIController): - @web.require_admin - @web.legacy_expose_api + @require_admin + @expose_api def index(self, trans, group_id, **kwd): """ GET /api/groups/{encoded_group_id}/roles @@ -41,8 +44,8 @@ class GroupRolesAPIController(BaseAPIController): trans.response.status = 500 return rval - @web.require_admin - @web.legacy_expose_api + @require_admin + @expose_api def show(self, trans, id, group_id, **kwd): """ GET /api/groups/{encoded_group_id}/roles/{encoded_role_id} @@ -67,8 +70,8 @@ class GroupRolesAPIController(BaseAPIController): log.error(item + ": %s", unicodify(e)) return item - @web.require_admin - @web.legacy_expose_api + @require_admin + @expose_api def update(self, trans, id, group_id, **kwd): """ PUT /api/groups/{encoded_group_id}/roles/{encoded_role_id} @@ -99,8 +102,8 @@ class GroupRolesAPIController(BaseAPIController): log.error(item + ": %s", unicodify(e)) return item - @web.require_admin - @web.legacy_expose_api + @require_admin + @expose_api def delete(self, trans, id, group_id, **kwd): """ DELETE /api/groups/{encoded_group_id}/roles/{encoded_role_id} From 3f49ea92133c68efe1499047ddf4a62c9bd90224 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Feb 2021 20:12:07 +0100 Subject: [PATCH 23/70] Move controller logic to GroupRolesManager --- lib/galaxy/managers/group_roles.py | 115 +++++++++++++++++++ lib/galaxy/webapps/galaxy/api/group_roles.py | 94 ++------------- 2 files changed, 125 insertions(+), 84 deletions(-) create mode 100644 lib/galaxy/managers/group_roles.py diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py new file mode 100644 index 00000000000..cc0218e6178 --- /dev/null +++ b/lib/galaxy/managers/group_roles.py @@ -0,0 +1,115 @@ +import logging + +from galaxy.util import unicodify +from galaxy.web import url_for + +log = logging.getLogger(__name__) + + +class GroupRolesManager: + """Interface/service object shared by controllers for interacting with group roles.""" + + def __init__(self, app) -> None: + self._app = app + + def index(self, trans, group_id): + """ + Returns a collection (list) of roles. + """ + decoded_group_id = trans.security.decode_id(group_id) + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + except Exception: + group = None + if not group: + trans.response.status = 400 + return "Invalid group id ( %s ) specified." % str(group_id) + rval = [] + try: + for gra in group.roles: + role = gra.role + encoded_id = trans.security.encode_id(role.id) + rval.append(dict(id=encoded_id, + name=role.name, + url=url_for('group_role', group_id=group_id, id=encoded_id, ))) + except Exception as e: + rval = "Error in group API at listing roles" + log.error(rval + ": %s", unicodify(e)) + trans.response.status = 500 + return rval + + def show(self, trans, id, group_id): + """ + Returns information about a group role. + """ + role_id = id + decoded_group_id = trans.security.decode_id(group_id) + decoded_role_id = trans.security.decode_id(role_id) + item = None + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) + for gra in group.roles: + if gra.role == role: + item = dict(id=role_id, + name=role.name, + url=url_for('group_role', group_id=group_id, id=role_id)) # TODO Fix This + if not item: + item = f"role {role.name} not in group {group.name}" + except Exception as e: + item = f"Error in group_role API group {group.name} role {role.name}" + log.error(item + ": %s", unicodify(e)) + return item + + def update(self, trans, id, group_id): + """ + Adds a role to a group + """ + role_id = id + decoded_group_id = trans.security.decode_id(group_id) + decoded_role_id = trans.security.decode_id(role_id) + item = None + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) + for gra in group.roles: + if gra.role == role: + item = dict(id=role_id, + name=role.name, + url=url_for('group_role', group_id=group_id, id=role_id)) + if not item: + gra = trans.app.model.GroupRoleAssociation(group, role) + # Add GroupRoleAssociation + trans.sa_session.add(gra) + trans.sa_session.flush() + item = dict(id=role_id, + name=role.name, + url=url_for('group_role', group_id=group_id, id=role_id)) + except Exception as e: + item = f"Error in group_role API Adding role {role.name} to group {group.name}" + log.error(item + ": %s", unicodify(e)) + return item + + def delete(self, trans, id, group_id): + """ + Removes a role from a group + """ + role_id = id + decoded_group_id = trans.security.decode_id(group_id) + decoded_role_id = trans.security.decode_id(role_id) + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) + for gra in group.roles: + if gra.role == role: + trans.sa_session.delete(gra) + trans.sa_session.flush() + item = dict(id=role_id, + name=role.name, + url=url_for('group_role', group_id=group_id, id=role_id)) + if not item: + item = f"role {role.name} not in group {group.name}" + except Exception as e: + item = f"Error in group_role API Removing role {role.name} from group {group.name}" + log.error(item + ": %s", unicodify(e)) + return item diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py index 3f944803dd6..d7361b5ec5c 100644 --- a/lib/galaxy/webapps/galaxy/api/group_roles.py +++ b/lib/galaxy/webapps/galaxy/api/group_roles.py @@ -3,18 +3,22 @@ API operations on Group objects. """ import logging -from galaxy.util import unicodify +from galaxy.managers.group_roles import GroupRolesManager from galaxy.web import ( expose_api, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController, url_for +from galaxy.webapps.base.controller import BaseAPIController log = logging.getLogger(__name__) class GroupRolesAPIController(BaseAPIController): + def __init__(self, app): + super().__init__(app) + self.manager = GroupRolesManager(app) + @require_admin @expose_api def index(self, trans, group_id, **kwd): @@ -22,27 +26,7 @@ class GroupRolesAPIController(BaseAPIController): GET /api/groups/{encoded_group_id}/roles Displays a collection (list) of groups. """ - decoded_group_id = trans.security.decode_id(group_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - except Exception: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str(group_id) - rval = [] - try: - for gra in group.roles: - role = gra.role - encoded_id = trans.security.encode_id(role.id) - rval.append(dict(id=encoded_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=encoded_id, ))) - except Exception as e: - rval = "Error in group API at listing roles" - log.error(rval + ": %s", unicodify(e)) - trans.response.status = 500 - return rval + return self.manager.index(trans, group_id) @require_admin @expose_api @@ -51,24 +35,7 @@ class GroupRolesAPIController(BaseAPIController): GET /api/groups/{encoded_group_id}/roles/{encoded_role_id} Displays information about a group role. """ - role_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_role_id = trans.security.decode_id(role_id) - item = None - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) - for gra in group.roles: - if gra.role == role: - item = dict(id=role_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=role_id)) # TODO Fix This - if not item: - item = f"role {role.name} not in group {group.name}" - except Exception as e: - item = f"Error in group_role API group {group.name} role {role.name}" - log.error(item + ": %s", unicodify(e)) - return item + return self.manager.show(trans, id, group_id) @require_admin @expose_api @@ -77,30 +44,7 @@ class GroupRolesAPIController(BaseAPIController): PUT /api/groups/{encoded_group_id}/roles/{encoded_role_id} Adds a role to a group """ - role_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_role_id = trans.security.decode_id(role_id) - item = None - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) - for gra in group.roles: - if gra.role == role: - item = dict(id=role_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=role_id)) - if not item: - gra = trans.app.model.GroupRoleAssociation(group, role) - # Add GroupRoleAssociation - trans.sa_session.add(gra) - trans.sa_session.flush() - item = dict(id=role_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=role_id)) - except Exception as e: - item = f"Error in group_role API Adding role {role.name} to group {group.name}" - log.error(item + ": %s", unicodify(e)) - return item + return self.manager.update(trans, id, group_id) @require_admin @expose_api @@ -109,22 +53,4 @@ class GroupRolesAPIController(BaseAPIController): DELETE /api/groups/{encoded_group_id}/roles/{encoded_role_id} Removes a role from a group """ - role_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_role_id = trans.security.decode_id(role_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) - for gra in group.roles: - if gra.role == role: - trans.sa_session.delete(gra) - trans.sa_session.flush() - item = dict(id=role_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=role_id)) - if not item: - item = f"role {role.name} not in group {group.name}" - except Exception as e: - item = f"Error in group_role API Removing role {role.name} from group {group.name}" - log.error(item + ": %s", unicodify(e)) - return item + return self.manager.delete(trans, id, group_id) From 26f731f060de198ec6ab1108f76cc2efa1e1dcf0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 19 Feb 2021 04:24:53 +0000 Subject: [PATCH 24/70] Fix sniffers of interval datatypes for files starting with many comments For files starting with >60 comment lines, the `sniff_prefix()` method of Bed, Gff, Gff3 and Gtf classes was returning `True` when actually all headers lines were skipped. Also: - Don't skip lines having more than one column just because the first column is empty. - Properly skip comment lines in `is_column_based()` - Add test file which would have been incorrectly sniffed as `gtf` before. It still isn't sniffed as `gff3`, but that's fine because it has too many comment lines for the default header prefix. --- lib/galaxy/datatypes/interval.py | 156 ++++++++++-------- lib/galaxy/datatypes/sniff.py | 29 ++-- .../test/too_many_comments_gff3.tabular | 65 ++++++++ 3 files changed, 165 insertions(+), 85 deletions(-) create mode 100644 lib/galaxy/datatypes/test/too_many_comments_gff3.tabular diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 89d771b42e6..20728b5d3d8 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -477,9 +477,9 @@ class Bed(Interval): if not get_headers(file_prefix, '\t', comment_designator='#', count=1): return False try: - headers = iter_headers(file_prefix, '\t', comment_designator='#') - for hdr in headers: - if hdr[0] == '': + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t', comment_designator='#'): + if not hdr or hdr == ['']: continue if len(hdr) < 3 or len(hdr) > 12: return False @@ -542,7 +542,8 @@ class Bed(Interval): return False if len(block_sizes) != block_count or len(block_starts) != block_count: return False - return True + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -818,28 +819,33 @@ class Gff(Tabular, _RemoteCallMixin): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: return False - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: - return False + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + int(hdr[4]) + except Exception: + return False + if hdr[5] != '.': try: - int(hdr[3]) - int(hdr[4]) + float(hdr[5]) except Exception: return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in data.valid_strand: - return False - if hdr[7] not in self.valid_gff_frame: - return False - return True + if hdr[6] not in data.valid_strand: + return False + if hdr[7] not in self.valid_gff_frame: + return False + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -953,37 +959,41 @@ class Gff3(Gff): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('3') >= 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('3') >= 0: return True - elif hdr and hdr[0].startswith('##gff-version') and hdr[0].find('3') < 0: + elif hdr[0].startswith('##gff-version') and hdr[0].find('3') < 0: return False - # Header comments may have been stripped, so inspect the data - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + except Exception: + if hdr[3] != '.': return False + try: + int(hdr[4]) + except Exception: + if hdr[4] != '.': + return False + if hdr[5] != '.': try: - int(hdr[3]) + float(hdr[5]) except Exception: - if hdr[3] != '.': - return False - try: - int(hdr[4]) - except Exception: - if hdr[4] != '.': - return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in self.valid_gff3_strand: return False - if hdr[7] not in self.valid_gff3_phase: - return False - parse_gff3_attributes(hdr[8]) - return True + if hdr[6] not in self.valid_gff3_strand: + return False + if hdr[7] not in self.valid_gff3_phase: + return False + parse_gff3_attributes(hdr[8]) + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -1031,34 +1041,38 @@ class Gtf(Gff): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: return False - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: - return False + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + int(hdr[4]) + except Exception: + return False + if hdr[5] != '.': try: - int(hdr[3]) - int(hdr[4]) + float(hdr[5]) except Exception: return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in data.valid_strand: - return False - if hdr[7] not in self.valid_gff_frame: - return False - - # Check attributes for gene_id (transcript_id is also mandatory - # but not for genes) - attributes = parse_gff_attributes(hdr[8]) - if 'gene_id' not in attributes: - return False - return True + if hdr[6] not in data.valid_strand: + return False + if hdr[7] not in self.valid_gff_frame: + return False + # Check attributes for gene_id (transcript_id is also mandatory + # but not for genes) + attributes = parse_gff_attributes(hdr[8]) + if 'gene_id' not in attributes: + return False + found_valid_lines = True + return found_valid_lines except Exception: return False diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 272042ece26..a63d5340bb9 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -235,24 +235,22 @@ def is_column_based(fname_or_file_prefix, sep='\t', skip=0): return False try: - headers = get_headers(fname_or_file_prefix, sep) + headers = get_headers(fname_or_file_prefix, sep, comment_designator='#')[skip:] except UnicodeDecodeError: return False count = 0 if not headers: return False - for hdr in headers[skip:]: - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) > 1: + for hdr in headers: + if hdr and hdr != ['']: + if count: + if len(hdr) != count: + return False + else: count = len(hdr) - break - if count < 2: - return False - for hdr in headers[skip:]: - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != count: - return False - return True + if count < 2: + return False + return count >= 2 def guess_ext(fname, sniff_order, is_binary=False): @@ -303,13 +301,13 @@ def guess_ext(fname, sniff_order, is_binary=False): >>> guess_ext(fname, sniff_order) 'gff3' >>> fname = get_test_fname('2.txt') - >>> guess_ext(fname, sniff_order) # 2.txt + >>> guess_ext(fname, sniff_order) 'txt' >>> fname = get_test_fname('2.tabular') >>> guess_ext(fname, sniff_order) 'tabular' >>> fname = get_test_fname('3.txt') - >>> guess_ext(fname, sniff_order) # 3.txt + >>> guess_ext(fname, sniff_order) 'txt' >>> fname = get_test_fname('test_tab1.tabular') >>> guess_ext(fname, sniff_order) @@ -451,6 +449,9 @@ def guess_ext(fname, sniff_order, is_binary=False): >>> fname = get_test_fname('1imzml') >>> guess_ext(fname, sniff_order) # This test case is ensuring doesn't throw exception, actual value could change if non-utf encoding handling improves. 'data' + >>> fname = get_test_fname('too_many_comments_gff3.tabular') + >>> guess_ext(fname, sniff_order) # It's a VCF but is sniffed as tabular because of the limit on the number of header lines we read + 'tabular' """ file_prefix = FilePrefix(fname) file_ext = run_sniffers_raw(file_prefix, sniff_order, is_binary) diff --git a/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular b/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular new file mode 100644 index 00000000000..e3fb5113dc6 --- /dev/null +++ b/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular @@ -0,0 +1,65 @@ +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +ctgA est match 5410 7503 . - . ID=EST:agt830.3;Target=agt830.3+1+595 +ctgA est HSP 7000 7503 . - . Parent=EST:agt830.3;Target=agt830.3+1+504 +ctgA est HSP 5410 5500 . ? . Parent=EST:agt830.3;Target=agt830.3+505+595 From ebf996f8bcd46c35807dd706ef8d35e1913b79d5 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 19 Feb 2021 04:56:37 +0000 Subject: [PATCH 25/70] Set Vcf metadata also for VcfGz files --- lib/galaxy/datatypes/tabular.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index a9513f9ccb0..ad23827bfc2 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -743,13 +743,12 @@ class BaseVcf(Tabular): def set_meta(self, dataset, **kwd): super().set_meta(dataset, **kwd) - source = open(dataset.file_name) - - # Skip comments. line = None - for line in source: - if not line.startswith('##'): - break + with compression_utils.get_fileobj(dataset.file_name) as fh: + # Skip comments. + for line in fh: + if not line.startswith('##'): + break if line and line.startswith('#'): # Found header line, get sample names. @@ -815,7 +814,7 @@ class VcfGz(BaseVcf, binary.Binary): return binascii.hexlify(last28) == b'1f8b08040000000000ff0600424302001b0003000000000000000000' def set_meta(self, dataset, **kwd): - super(BaseVcf, self).set_meta(dataset, **kwd) + super().set_meta(dataset, **kwd) """ Creates the index for the VCF file. """ # These metadata values are not accessible by users, always overwrite index_file = dataset.metadata.tabix_index From f880010b51457a5fee6b37b1827bb3e5af7a99b5 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Wed, 17 Feb 2021 22:28:47 +0200 Subject: [PATCH 26/70] implement selenium test --- .../LibraryFolderDatasetPermissions.vue | 2 +- .../PermissionsInputField.vue | 2 +- lib/galaxy/selenium/navigation.yml | 4 +++ lib/galaxy_test/selenium/framework.py | 1 + ...ir.py => test_user_library_permissions.py} | 31 ++++++++++++++++--- 5 files changed, 34 insertions(+), 6 deletions(-) rename test/integration_selenium/{test_user_library_import_dir.py => test_user_library_permissions.py} (70%) diff --git a/client/src/components/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue b/client/src/components/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue index c7f5b2050d8..89233cc2973 100644 --- a/client/src/components/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue +++ b/client/src/components/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue @@ -61,7 +61,7 @@

You can - make this dataset private + make this dataset private to you.

diff --git a/client/src/components/LibraryFolder/LibraryFolderPermissions/PermissionsInputField.vue b/client/src/components/LibraryFolder/LibraryFolderPermissions/PermissionsInputField.vue index c5125c8061d..cfcb571bcb3 100644 --- a/client/src/components/LibraryFolder/LibraryFolderPermissions/PermissionsInputField.vue +++ b/client/src/components/LibraryFolder/LibraryFolderPermissions/PermissionsInputField.vue @@ -5,7 +5,7 @@ -
+
Date: Fri, 19 Feb 2021 12:59:56 +0100 Subject: [PATCH 27/70] Refactor manager logic - Adapt error handling - Remove redundant exception logging --- lib/galaxy/managers/group_roles.py | 170 ++++++++++++++--------------- 1 file changed, 84 insertions(+), 86 deletions(-) diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py index cc0218e6178..55ad0475729 100644 --- a/lib/galaxy/managers/group_roles.py +++ b/lib/galaxy/managers/group_roles.py @@ -1,6 +1,19 @@ import logging +from typing import ( + Any, + Dict, + List, + Optional, +) -from galaxy.util import unicodify +from galaxy import model +from galaxy.app import StructuredApp +from galaxy.exceptions import ( + ObjectNotFound, +) +from galaxy.managers.base import decode_id +from galaxy.managers.context import ProvidesAppContext +from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.web import url_for log = logging.getLogger(__name__) @@ -9,107 +22,92 @@ log = logging.getLogger(__name__) class GroupRolesManager: """Interface/service object shared by controllers for interacting with group roles.""" - def __init__(self, app) -> None: + def __init__(self, app: StructuredApp) -> None: self._app = app - def index(self, trans, group_id): + def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]: """ - Returns a collection (list) of roles. + Returns a collection roles associated with the given group. """ - decoded_group_id = trans.security.decode_id(group_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - except Exception: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str(group_id) + group = self._get_group(trans, group_id) rval = [] - try: - for gra in group.roles: - role = gra.role - encoded_id = trans.security.encode_id(role.id) - rval.append(dict(id=encoded_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=encoded_id, ))) - except Exception as e: - rval = "Error in group API at listing roles" - log.error(rval + ": %s", unicodify(e)) - trans.response.status = 500 + for gra in group.roles: + group_role = self._serialize_group_role(group_id, gra.role) + rval.append(group_role) return rval - def show(self, trans, id, group_id): + def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> Dict[str, Any]: """ Returns information about a group role. """ role_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_role_id = trans.security.decode_id(role_id) - item = None - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) - for gra in group.roles: - if gra.role == role: - item = dict(id=role_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=role_id)) # TODO Fix This - if not item: - item = f"role {role.name} not in group {group.name}" - except Exception as e: - item = f"Error in group_role API group {group.name} role {role.name}" - log.error(item + ": %s", unicodify(e)) - return item + group = self._get_group(trans, group_id) + role = self._get_role(trans, role_id) + group_role = self._get_group_role(trans, group, role) + if group_role is None: + raise ObjectNotFound(f"Role {role.name} not in group {group.name}") - def update(self, trans, id, group_id): + return self._serialize_group_role(group_id, role) + + def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField): """ - Adds a role to a group + Adds a role to a group if it is not already associated. """ role_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_role_id = trans.security.decode_id(role_id) - item = None - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) - for gra in group.roles: - if gra.role == role: - item = dict(id=role_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=role_id)) - if not item: - gra = trans.app.model.GroupRoleAssociation(group, role) - # Add GroupRoleAssociation - trans.sa_session.add(gra) - trans.sa_session.flush() - item = dict(id=role_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=role_id)) - except Exception as e: - item = f"Error in group_role API Adding role {role.name} to group {group.name}" - log.error(item + ": %s", unicodify(e)) - return item + group = self._get_group(trans, group_id) + role = self._get_role(trans, role_id) + group_role = self._get_group_role(trans, group, role) + if group_role is None: + self._add_role_to_group(trans, group, role) - def delete(self, trans, id, group_id): + return self._serialize_group_role(group_id, role) + + def delete(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField): """ - Removes a role from a group + Removes a role from a group. """ role_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_role_id = trans.security.decode_id(role_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - role = trans.sa_session.query(trans.app.model.Role).get(decoded_role_id) - for gra in group.roles: - if gra.role == role: - trans.sa_session.delete(gra) - trans.sa_session.flush() - item = dict(id=role_id, - name=role.name, - url=url_for('group_role', group_id=group_id, id=role_id)) - if not item: - item = f"role {role.name} not in group {group.name}" - except Exception as e: - item = f"Error in group_role API Removing role {role.name} from group {group.name}" - log.error(item + ": %s", unicodify(e)) - return item + group = self._get_group(trans, group_id) + role = self._get_role(trans, role_id) + group_role = self._get_group_role(trans, group, role) + if group_role is None: + raise ObjectNotFound(f"Role {role.name} not in group {group.name}") + self._remove_role_from_group(trans, group_role) + return self._serialize_group_role(group_id, role) + + def _get_group(self, trans: ProvidesAppContext, encoded_group_id: EncodedDatabaseIdField) -> Any: + decoded_group_id = decode_id(self._app, encoded_group_id) + group = trans.sa_session.query(model.Group).get(decoded_group_id) + if group is None: + raise ObjectNotFound(f"Group with id {encoded_group_id} was not found.") + return group + + def _get_role(self, trans: ProvidesAppContext, encoded_role_id: EncodedDatabaseIdField) -> model.Role: + decoded_role_id = decode_id(self._app, encoded_role_id) + role = trans.sa_session.query(model.Role).get(decoded_role_id) + if role is None: + raise ObjectNotFound(f"Role with id {encoded_role_id} was not found.") + return role + + def _get_group_role(self, trans: ProvidesAppContext, group: model.Group, role: model.Role) -> Optional[model.GroupRoleAssociation]: + return trans.sa_session.query(model.GroupRoleAssociation).filter( + model.GroupRoleAssociation.group == group, + model.GroupRoleAssociation.role == role + ).one_or_none() + + def _add_role_to_group(self, trans: ProvidesAppContext, group: model.Group, role: model.Role): + gra = model.GroupRoleAssociation(group, role) + trans.sa_session.add(gra) + trans.sa_session.flush() + + def _remove_role_from_group(self, trans: ProvidesAppContext, group_role: model.GroupRoleAssociation): + trans.sa_session.delete(group_role) + trans.sa_session.flush() + + def _serialize_group_role(self, encoded_group_id: EncodedDatabaseIdField, role: model.Role): + encoded_role_id = self._app.security.encode_id(role.id) + return { + "id": encoded_role_id, + "name": role.name, + "url": url_for('group_role', group_id=encoded_group_id, id=encoded_role_id) + } From a1c2a77ef311590f63ec948bf3750338b397c0f3 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Feb 2021 13:00:14 +0100 Subject: [PATCH 28/70] Add type annotations to controller --- lib/galaxy/webapps/galaxy/api/group_roles.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py index d7361b5ec5c..3be7ed95880 100644 --- a/lib/galaxy/webapps/galaxy/api/group_roles.py +++ b/lib/galaxy/webapps/galaxy/api/group_roles.py @@ -3,7 +3,9 @@ API operations on Group objects. """ import logging +from galaxy.managers.context import ProvidesAppContext from galaxy.managers.group_roles import GroupRolesManager +from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.web import ( expose_api, require_admin, @@ -21,7 +23,7 @@ class GroupRolesAPIController(BaseAPIController): @require_admin @expose_api - def index(self, trans, group_id, **kwd): + def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField, **kwd): """ GET /api/groups/{encoded_group_id}/roles Displays a collection (list) of groups. @@ -30,7 +32,7 @@ class GroupRolesAPIController(BaseAPIController): @require_admin @expose_api - def show(self, trans, id, group_id, **kwd): + def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField, **kwd): """ GET /api/groups/{encoded_group_id}/roles/{encoded_role_id} Displays information about a group role. @@ -39,7 +41,7 @@ class GroupRolesAPIController(BaseAPIController): @require_admin @expose_api - def update(self, trans, id, group_id, **kwd): + def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField, **kwd): """ PUT /api/groups/{encoded_group_id}/roles/{encoded_role_id} Adds a role to a group @@ -48,7 +50,7 @@ class GroupRolesAPIController(BaseAPIController): @require_admin @expose_api - def delete(self, trans, id, group_id, **kwd): + def delete(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField, **kwd): """ DELETE /api/groups/{encoded_group_id}/roles/{encoded_role_id} Removes a role from a group From 470c658eba5c2cd0e89fe337d1d4367f83541f96 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Feb 2021 14:22:02 +0100 Subject: [PATCH 29/70] Add tests for group_users API --- lib/galaxy_test/api/test_group_users.py | 135 ++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 lib/galaxy_test/api/test_group_users.py diff --git a/lib/galaxy_test/api/test_group_users.py b/lib/galaxy_test/api/test_group_users.py new file mode 100644 index 00000000000..2714a66cd40 --- /dev/null +++ b/lib/galaxy_test/api/test_group_users.py @@ -0,0 +1,135 @@ +from typing import List + +from galaxy_test.base.populators import DatasetPopulator +from ._framework import ApiTestCase + + +class GroupUsersApiTestCase(ApiTestCase): + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + + def test_index(self, group_name: str = None): + group_name = group_name or "test-group" + group = self._create_group(group_name) + encoded_group_id = group["id"] + + group_users = self._get_group_users(encoded_group_id) + + assert isinstance(group_users, list) + assert len(group_users) > 0 + for group_user in group_users: + self._assert_valid_group_user(group_user) + + def test_index_only_admin(self): + encoded_group_id = "any-group-id" + response = self._get(f"groups/{encoded_group_id}/users") + self._assert_status_code_is(response, 403) + + def test_index_unknown_group_raises_400(self): + encoded_group_id = "unknown-group-id" + response = self._get(f"groups/{encoded_group_id}/users", admin=True) + self._assert_status_code_is(response, 400) + + def test_show(self): + encoded_user_id = self.dataset_populator.user_id() + group = self._create_group("test-group-show-user", encoded_user_ids=[encoded_user_id]) + encoded_group_id = group["id"] + response = self._get(f"groups/{encoded_group_id}/users/{encoded_user_id}", admin=True) + self._assert_status_code_is(response, 200) + group_user = response.json() + self._assert_valid_group_user(group_user) + + def test_show_only_admin(self): + encoded_group_id = "any-group-id" + encoded_user_id = "any-user-id" + response = self._get(f"groups/{encoded_group_id}/users/{encoded_user_id}") + self._assert_status_code_is(response, 403) + + def test_show_unknown_raises_400(self): + group = self._create_group("test-group-with-unknown-user") + encoded_group_id = group["id"] + encoded_user_id = "unknown-user-id" + response = self._get(f"groups/{encoded_group_id}/users/{encoded_user_id}", admin=True) + self._assert_status_code_is(response, 400) + + def test_update(self): + group_name = "group-without-users" + group = self._create_group(group_name, encoded_user_ids=[]) + encoded_group_id = group["id"] + + group_users = self._get_group_users(encoded_group_id) + assert len(group_users) == 0 + + encoded_user_id = self.dataset_populator.user_id() + update_response = self._put(f"groups/{encoded_group_id}/users/{encoded_user_id}", admin=True) + self._assert_status_code_is_ok(update_response) + group_user = update_response.json() + self._assert_valid_group_user(group_user, assert_id=encoded_user_id) + assert group_user["url"] == f"/api/groups/{encoded_group_id}/users/{encoded_user_id}" + + def test_update_only_admin(self): + encoded_group_id = "any-group-id" + encoded_user_id = "any-user-id" + response = self._put(f"groups/{encoded_group_id}/users/{encoded_user_id}") + self._assert_status_code_is(response, 403) + + def test_delete(self): + group_name = "group-with-user-to-delete" + encoded_user_id = self.dataset_populator.user_id() + group = self._create_group(group_name, encoded_user_ids=[encoded_user_id]) + encoded_group_id = group["id"] + + group_users = self._get_group_users(encoded_group_id) + assert len(group_users) == 1 + + delete_response = self._delete(f"groups/{encoded_group_id}/users/{encoded_user_id}", admin=True) + self._assert_status_code_is_ok(delete_response) + group_user = delete_response.json() + self._assert_valid_group_user(group_user, assert_id=encoded_user_id) + + group_users = self._get_group_users(encoded_group_id) + assert len(group_users) == 0 + + def test_delete_only_admin(self): + encoded_group_id = "any-group-id" + encoded_user_id = "any-user-id" + response = self._delete(f"groups/{encoded_group_id}/users/{encoded_user_id}") + self._assert_status_code_is(response, 403) + + def test_delete_unknown_raises_400(self): + group_name = "group-without-user-to-delete" + group = self._create_group(group_name, encoded_user_ids=[]) + encoded_group_id = group["id"] + + group_users = self._get_group_users(encoded_group_id) + assert len(group_users) == 0 + + encoded_user_id = "unknown-user-id" + delete_response = self._delete(f"groups/{encoded_group_id}/users/{encoded_user_id}", admin=True) + self._assert_status_code_is(delete_response, 400) + + def _create_group(self, group_name: str, encoded_user_ids: List[str] = None): + if encoded_user_ids is None: + encoded_user_ids = [self.dataset_populator.user_id()] + user_ids = encoded_user_ids + payload = { + "name": group_name, + "user_ids": user_ids, + } + response = self._post("groups", payload, admin=True, json=True) + self._assert_status_code_is(response, 200) + group = response.json()[0] # POST /api/groups returns a list + return group + + def _get_group_users(self, encoded_group_id: str): + response = self._get(f"groups/{encoded_group_id}/users", admin=True) + self._assert_status_code_is(response, 200) + group_users = response.json() + return group_users + + def _assert_valid_group_user(self, user, assert_id=None): + self._assert_has_keys(user, "id", "email", "url") + if assert_id is not None: + assert user["id"] == assert_id From e3783deb17dbd608d55fb280d32bb8488f547cb3 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Feb 2021 14:25:28 +0100 Subject: [PATCH 30/70] Replace the old legacy_expose_api decorators --- lib/galaxy/webapps/galaxy/api/group_users.py | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/group_users.py b/lib/galaxy/webapps/galaxy/api/group_users.py index b978b8617d2..123d8640402 100644 --- a/lib/galaxy/webapps/galaxy/api/group_users.py +++ b/lib/galaxy/webapps/galaxy/api/group_users.py @@ -3,8 +3,11 @@ API operations on Group objects. """ import logging -from galaxy import web from galaxy.util import unicodify +from galaxy.web import ( + expose_api, + require_admin, +) from galaxy.webapps.base.controller import BaseAPIController, url_for log = logging.getLogger(__name__) @@ -12,8 +15,8 @@ log = logging.getLogger(__name__) class GroupUsersAPIController(BaseAPIController): - @web.require_admin - @web.legacy_expose_api + @require_admin + @expose_api def index(self, trans, group_id, **kwd): """ GET /api/groups/{encoded_group_id}/users @@ -41,8 +44,8 @@ class GroupUsersAPIController(BaseAPIController): trans.response.status = 500 return rval - @web.require_admin - @web.legacy_expose_api + @require_admin + @expose_api def show(self, trans, id, group_id, **kwd): """ GET /api/groups/{encoded_group_id}/users/{encoded_user_id} @@ -67,8 +70,8 @@ class GroupUsersAPIController(BaseAPIController): log.error(item + ": %s", unicodify(e)) return item - @web.require_admin - @web.legacy_expose_api + @require_admin + @expose_api def update(self, trans, id, group_id, **kwd): """ PUT /api/groups/{encoded_group_id}/users/{encoded_user_id} @@ -99,8 +102,8 @@ class GroupUsersAPIController(BaseAPIController): log.error(item + ": %s", unicodify(e)) return item - @web.require_admin - @web.legacy_expose_api + @require_admin + @expose_api def delete(self, trans, id, group_id, **kwd): """ DELETE /api/groups/{encoded_group_id}/users/{encoded_user_id} From b5988a87777ae4449add6f06f516a57d91d2a744 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Feb 2021 14:33:32 +0100 Subject: [PATCH 31/70] Move controller logic to GroupUsersManager --- lib/galaxy/managers/group_users.py | 115 +++++++++++++++++++ lib/galaxy/webapps/galaxy/api/group_users.py | 94 ++------------- 2 files changed, 125 insertions(+), 84 deletions(-) create mode 100644 lib/galaxy/managers/group_users.py diff --git a/lib/galaxy/managers/group_users.py b/lib/galaxy/managers/group_users.py new file mode 100644 index 00000000000..e9e995bb8c2 --- /dev/null +++ b/lib/galaxy/managers/group_users.py @@ -0,0 +1,115 @@ +import logging + +from galaxy.util import unicodify +from galaxy.web import url_for + +log = logging.getLogger(__name__) + + +class GroupUsersManager: + """Interface/service object shared by controllers for interacting with group users.""" + + def __init__(self, app) -> None: + self._app = app + + def index(self, trans, group_id, **kwd): + """ + Displays a collection (list) of groups. + """ + decoded_group_id = trans.security.decode_id(group_id) + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + except Exception: + group = None + if not group: + trans.response.status = 400 + return "Invalid group id ( %s ) specified." % str(group_id) + rval = [] + try: + for uga in group.users: + user = uga.user + encoded_id = trans.security.encode_id(user.id) + rval.append(dict(id=encoded_id, + email=user.email, + url=url_for('group_user', group_id=group_id, id=encoded_id, ))) + except Exception as e: + rval = "Error in group API at listing users" + log.error(rval + ": %s", unicodify(e)) + trans.response.status = 500 + return rval + + def show(self, trans, id, group_id, **kwd): + """ + Displays information about a group user. + """ + user_id = id + decoded_group_id = trans.security.decode_id(group_id) + decoded_user_id = trans.security.decode_id(user_id) + item = None + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) + for uga in group.users: + if uga.user == user: + item = dict(id=user_id, + email=user.email, + url=url_for('group_user', group_id=group_id, id=user_id)) # TODO Fix This + if not item: + item = f"user {user.email} not in group {group.name}" + except Exception as e: + item = f"Error in group_user API group {group.name} user {user.email}" + log.error(item + ": %s", unicodify(e)) + return item + + def update(self, trans, id, group_id, **kwd): + """ + Adds a user to a group + """ + user_id = id + decoded_group_id = trans.security.decode_id(group_id) + decoded_user_id = trans.security.decode_id(user_id) + item = None + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) + for uga in group.users: + if uga.user == user: + item = dict(id=user_id, + email=user.email, + url=url_for('group_user', group_id=group_id, id=user_id)) + if not item: + uga = trans.app.model.UserGroupAssociation(user, group) + # Add UserGroupAssociations + trans.sa_session.add(uga) + trans.sa_session.flush() + item = dict(id=user_id, + email=user.email, + url=url_for('group_user', group_id=group_id, id=user_id)) + except Exception as e: + item = f"Error in group_user API Adding user {user.email} to group {group.name}" + log.error(item + ": %s", unicodify(e)) + return item + + def delete(self, trans, id, group_id, **kwd): + """ + Removes a user from a group + """ + user_id = id + decoded_group_id = trans.security.decode_id(group_id) + decoded_user_id = trans.security.decode_id(user_id) + try: + group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) + user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) + for uga in group.users: + if uga.user == user: + trans.sa_session.delete(uga) + trans.sa_session.flush() + item = dict(id=user_id, + email=user.email, + url=url_for('group_user', group_id=group_id, id=user_id)) + if not item: + item = f"user {user.email} not in group {group.name}" + except Exception as e: + item = f"Error in group_user API Removing user {user.email} from group {group.name}" + log.error(item + ": %s", unicodify(e)) + return item diff --git a/lib/galaxy/webapps/galaxy/api/group_users.py b/lib/galaxy/webapps/galaxy/api/group_users.py index 123d8640402..f33499263aa 100644 --- a/lib/galaxy/webapps/galaxy/api/group_users.py +++ b/lib/galaxy/webapps/galaxy/api/group_users.py @@ -3,18 +3,22 @@ API operations on Group objects. """ import logging -from galaxy.util import unicodify +from galaxy.managers.group_users import GroupUsersManager from galaxy.web import ( expose_api, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController, url_for +from galaxy.webapps.base.controller import BaseAPIController log = logging.getLogger(__name__) class GroupUsersAPIController(BaseAPIController): + def __init__(self, app): + super().__init__(app) + self.manager = GroupUsersManager(app) + @require_admin @expose_api def index(self, trans, group_id, **kwd): @@ -22,27 +26,7 @@ class GroupUsersAPIController(BaseAPIController): GET /api/groups/{encoded_group_id}/users Displays a collection (list) of groups. """ - decoded_group_id = trans.security.decode_id(group_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - except Exception: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str(group_id) - rval = [] - try: - for uga in group.users: - user = uga.user - encoded_id = trans.security.encode_id(user.id) - rval.append(dict(id=encoded_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=encoded_id, ))) - except Exception as e: - rval = "Error in group API at listing users" - log.error(rval + ": %s", unicodify(e)) - trans.response.status = 500 - return rval + return self.manager.index(trans, group_id) @require_admin @expose_api @@ -51,24 +35,7 @@ class GroupUsersAPIController(BaseAPIController): GET /api/groups/{encoded_group_id}/users/{encoded_user_id} Displays information about a group user. """ - user_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_user_id = trans.security.decode_id(user_id) - item = None - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) - for uga in group.users: - if uga.user == user: - item = dict(id=user_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=user_id)) # TODO Fix This - if not item: - item = f"user {user.email} not in group {group.name}" - except Exception as e: - item = f"Error in group_user API group {group.name} user {user.email}" - log.error(item + ": %s", unicodify(e)) - return item + return self.manager.show(trans, id, group_id) @require_admin @expose_api @@ -77,30 +44,7 @@ class GroupUsersAPIController(BaseAPIController): PUT /api/groups/{encoded_group_id}/users/{encoded_user_id} Adds a user to a group """ - user_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_user_id = trans.security.decode_id(user_id) - item = None - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) - for uga in group.users: - if uga.user == user: - item = dict(id=user_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=user_id)) - if not item: - uga = trans.app.model.UserGroupAssociation(user, group) - # Add UserGroupAssociations - trans.sa_session.add(uga) - trans.sa_session.flush() - item = dict(id=user_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=user_id)) - except Exception as e: - item = f"Error in group_user API Adding user {user.email} to group {group.name}" - log.error(item + ": %s", unicodify(e)) - return item + return self.manager.update(trans, id, group_id) @require_admin @expose_api @@ -109,22 +53,4 @@ class GroupUsersAPIController(BaseAPIController): DELETE /api/groups/{encoded_group_id}/users/{encoded_user_id} Removes a user from a group """ - user_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_user_id = trans.security.decode_id(user_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) - for uga in group.users: - if uga.user == user: - trans.sa_session.delete(uga) - trans.sa_session.flush() - item = dict(id=user_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=user_id)) - if not item: - item = f"user {user.email} not in group {group.name}" - except Exception as e: - item = f"Error in group_user API Removing user {user.email} from group {group.name}" - log.error(item + ": %s", unicodify(e)) - return item + return self.manager.delete(trans, id, group_id) From f77ba2cdc2b34b6d3f342e4ac4afa70ffb50be3b Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Feb 2021 15:08:47 +0100 Subject: [PATCH 32/70] Refactor manager logic - Adapt error handling - Remove redundant exception logging --- lib/galaxy/managers/group_users.py | 178 ++++++++++++++--------------- 1 file changed, 88 insertions(+), 90 deletions(-) diff --git a/lib/galaxy/managers/group_users.py b/lib/galaxy/managers/group_users.py index e9e995bb8c2..9dd194311ef 100644 --- a/lib/galaxy/managers/group_users.py +++ b/lib/galaxy/managers/group_users.py @@ -1,6 +1,19 @@ import logging +from typing import ( + Any, + Dict, + List, + Optional, +) -from galaxy.util import unicodify +from galaxy import model +from galaxy.app import StructuredApp +from galaxy.exceptions import ( + ObjectNotFound, +) +from galaxy.managers.base import decode_id +from galaxy.managers.context import ProvidesAppContext +from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.web import url_for log = logging.getLogger(__name__) @@ -9,107 +22,92 @@ log = logging.getLogger(__name__) class GroupUsersManager: """Interface/service object shared by controllers for interacting with group users.""" - def __init__(self, app) -> None: + def __init__(self, app: StructuredApp) -> None: self._app = app - def index(self, trans, group_id, **kwd): + def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]: """ - Displays a collection (list) of groups. + Returns a collection (list) with some information about users associated with the given group. """ - decoded_group_id = trans.security.decode_id(group_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - except Exception: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str(group_id) + group = self._get_group(trans, group_id) rval = [] - try: - for uga in group.users: - user = uga.user - encoded_id = trans.security.encode_id(user.id) - rval.append(dict(id=encoded_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=encoded_id, ))) - except Exception as e: - rval = "Error in group API at listing users" - log.error(rval + ": %s", unicodify(e)) - trans.response.status = 500 + for uga in group.users: + group_user = self._serialize_group_user(group_id, uga.user) + rval.append(group_user) return rval - def show(self, trans, id, group_id, **kwd): + def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> Dict[str, Any]: """ - Displays information about a group user. + Returns information about a group user. """ user_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_user_id = trans.security.decode_id(user_id) - item = None - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) - for uga in group.users: - if uga.user == user: - item = dict(id=user_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=user_id)) # TODO Fix This - if not item: - item = f"user {user.email} not in group {group.name}" - except Exception as e: - item = f"Error in group_user API group {group.name} user {user.email}" - log.error(item + ": %s", unicodify(e)) - return item + group = self._get_group(trans, group_id) + user = self._get_user(trans, user_id) + group_user = self._get_group_user(trans, group, user) + if group_user is None: + raise ObjectNotFound(f"User {user.email} not in group {group.name}") - def update(self, trans, id, group_id, **kwd): - """ - Adds a user to a group - """ - user_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_user_id = trans.security.decode_id(user_id) - item = None - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) - for uga in group.users: - if uga.user == user: - item = dict(id=user_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=user_id)) - if not item: - uga = trans.app.model.UserGroupAssociation(user, group) - # Add UserGroupAssociations - trans.sa_session.add(uga) - trans.sa_session.flush() - item = dict(id=user_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=user_id)) - except Exception as e: - item = f"Error in group_user API Adding user {user.email} to group {group.name}" - log.error(item + ": %s", unicodify(e)) - return item + return self._serialize_group_user(group_id, user) - def delete(self, trans, id, group_id, **kwd): + def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField): """ - Removes a user from a group + Adds a user to a group. """ user_id = id - decoded_group_id = trans.security.decode_id(group_id) - decoded_user_id = trans.security.decode_id(user_id) - try: - group = trans.sa_session.query(trans.app.model.Group).get(decoded_group_id) - user = trans.sa_session.query(trans.app.model.User).get(decoded_user_id) - for uga in group.users: - if uga.user == user: - trans.sa_session.delete(uga) - trans.sa_session.flush() - item = dict(id=user_id, - email=user.email, - url=url_for('group_user', group_id=group_id, id=user_id)) - if not item: - item = f"user {user.email} not in group {group.name}" - except Exception as e: - item = f"Error in group_user API Removing user {user.email} from group {group.name}" - log.error(item + ": %s", unicodify(e)) - return item + group = self._get_group(trans, group_id) + user = self._get_user(trans, user_id) + group_user = self._get_group_user(trans, group, user) + if group_user is None: + self._add_user_to_group(trans, group, user) + + return self._serialize_group_user(group_id, user) + + def delete(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField): + """ + Removes a user from a group. + """ + user_id = id + group = self._get_group(trans, group_id) + user = self._get_user(trans, user_id) + group_user = self._get_group_user(trans, group, user) + if group_user is None: + raise ObjectNotFound(f"User {user.email} not in group {group.name}") + self._remove_user_from_group(trans, group_user) + return self._serialize_group_user(group_id, user) + + def _get_group(self, trans: ProvidesAppContext, encoded_group_id: EncodedDatabaseIdField) -> Any: + decoded_group_id = decode_id(self._app, encoded_group_id) + group = trans.sa_session.query(model.Group).get(decoded_group_id) + if group is None: + raise ObjectNotFound(f"Group with id {encoded_group_id} was not found.") + return group + + def _get_user(self, trans: ProvidesAppContext, encoded_user_id: EncodedDatabaseIdField) -> model.User: + decoded_user_id = decode_id(self._app, encoded_user_id) + user = trans.sa_session.query(model.User).get(decoded_user_id) + if user is None: + raise ObjectNotFound(f"User with id {encoded_user_id} was not found.") + return user + + def _get_group_user(self, trans: ProvidesAppContext, group: model.Group, user: model.User) -> Optional[model.UserGroupAssociation]: + return trans.sa_session.query(model.UserGroupAssociation).filter( + model.UserGroupAssociation.user == user, + model.UserGroupAssociation.group == group + ).one_or_none() + + def _add_user_to_group(self, trans: ProvidesAppContext, group: model.Group, user: model.User): + gra = model.UserGroupAssociation(user, group) + trans.sa_session.add(gra) + trans.sa_session.flush() + + def _remove_user_from_group(self, trans: ProvidesAppContext, group_user: model.UserGroupAssociation): + trans.sa_session.delete(group_user) + trans.sa_session.flush() + + def _serialize_group_user(self, encoded_group_id: EncodedDatabaseIdField, user: model.User): + encoded_user_id = self._app.security.encode_id(user.id) + return { + "id": encoded_user_id, + "email": user.email, + "url": url_for('group_user', group_id=encoded_group_id, id=encoded_user_id) + } From 0b9e1bb4b4a375a4cdc4674af171aa57064c6bbc Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Feb 2021 15:09:04 +0100 Subject: [PATCH 33/70] Add type annotations to controller --- lib/galaxy/webapps/galaxy/api/group_users.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/group_users.py b/lib/galaxy/webapps/galaxy/api/group_users.py index f33499263aa..b1fb16b3235 100644 --- a/lib/galaxy/webapps/galaxy/api/group_users.py +++ b/lib/galaxy/webapps/galaxy/api/group_users.py @@ -3,7 +3,9 @@ API operations on Group objects. """ import logging +from galaxy.managers.context import ProvidesAppContext from galaxy.managers.group_users import GroupUsersManager +from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.web import ( expose_api, require_admin, @@ -21,7 +23,7 @@ class GroupUsersAPIController(BaseAPIController): @require_admin @expose_api - def index(self, trans, group_id, **kwd): + def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField, **kwd): """ GET /api/groups/{encoded_group_id}/users Displays a collection (list) of groups. @@ -30,7 +32,7 @@ class GroupUsersAPIController(BaseAPIController): @require_admin @expose_api - def show(self, trans, id, group_id, **kwd): + def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField, **kwd): """ GET /api/groups/{encoded_group_id}/users/{encoded_user_id} Displays information about a group user. @@ -39,7 +41,7 @@ class GroupUsersAPIController(BaseAPIController): @require_admin @expose_api - def update(self, trans, id, group_id, **kwd): + def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField, **kwd): """ PUT /api/groups/{encoded_group_id}/users/{encoded_user_id} Adds a user to a group @@ -48,7 +50,7 @@ class GroupUsersAPIController(BaseAPIController): @require_admin @expose_api - def delete(self, trans, id, group_id, **kwd): + def delete(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField, **kwd): """ DELETE /api/groups/{encoded_group_id}/users/{encoded_user_id} Removes a user from a group From f3f6c26470c044254efc895e02c707452e70b87a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 19 Feb 2021 15:29:46 -0500 Subject: [PATCH 34/70] Pin mypy to 0.790 --- lib/galaxy/dependencies/lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/lint-requirements.txt b/lib/galaxy/dependencies/lint-requirements.txt index 98ec924ae6d..6357c04e6d3 100644 --- a/lib/galaxy/dependencies/lint-requirements.txt +++ b/lib/galaxy/dependencies/lint-requirements.txt @@ -1,4 +1,4 @@ flake8 flake8-bugbear flake8-import-order -mypy +mypy==0.790 From e6cdbb8497da43fcb042a8f8d32baaac523e11b7 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 19 Feb 2021 22:31:55 -0500 Subject: [PATCH 35/70] Rename fixtures to fix test db object name conflict --- lib/galaxy_test/api/test_group_roles.py | 2 +- lib/galaxy_test/api/test_group_users.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy_test/api/test_group_roles.py b/lib/galaxy_test/api/test_group_roles.py index f0c5450fc36..67b00527ec3 100644 --- a/lib/galaxy_test/api/test_group_roles.py +++ b/lib/galaxy_test/api/test_group_roles.py @@ -11,7 +11,7 @@ class GroupRolesApiTestCase(ApiTestCase): self.dataset_populator = DatasetPopulator(self.galaxy_interactor) def test_index(self, group_name: str = None): - group_name = group_name or "test-group" + group_name = group_name or "test-group_roles" group = self._create_group(group_name) encoded_group_id = group["id"] diff --git a/lib/galaxy_test/api/test_group_users.py b/lib/galaxy_test/api/test_group_users.py index 2714a66cd40..7f1045e6851 100644 --- a/lib/galaxy_test/api/test_group_users.py +++ b/lib/galaxy_test/api/test_group_users.py @@ -11,7 +11,7 @@ class GroupUsersApiTestCase(ApiTestCase): self.dataset_populator = DatasetPopulator(self.galaxy_interactor) def test_index(self, group_name: str = None): - group_name = group_name or "test-group" + group_name = group_name or "test-group_users" group = self._create_group(group_name) encoded_group_id = group["id"] From 8a9b213822be2444bd229994722fb4bff92a5f53 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 20 Feb 2021 09:15:37 +0100 Subject: [PATCH 36/70] Let ModelMapping manage session scope --- lib/galaxy/dependencies/dev-requirements.txt | 1 + .../dependencies/pinned-requirements.txt | 1 + lib/galaxy/model/base.py | 37 ++++++++++++------- pyproject.toml | 1 + 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 710dd4864f1..12308a24d3c 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -38,6 +38,7 @@ cmd2==1.4.0; python_version >= "3.6" colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") coloredlogs==15.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" commonmark==0.9.1 +contextvars==2.4; python_version >= "3.6" and python_version < "3.7" coverage==5.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" cryptography==3.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cwltool==3.0.20201109103151; python_version >= "3.6" and python_version < "4" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index cb92da6600d..0efe6d663d8 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -35,6 +35,7 @@ cloudbridge==2.1.0 cmd2==1.4.0; python_version >= "3.6" colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" coloredlogs==15.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" +contextvars==2.4; python_version >= "3.6" and python_version < "3.7" cryptography==3.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cwltool==3.0.20201109103151; python_version >= "3.6" and python_version < "4" dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" diff --git a/lib/galaxy/model/base.py b/lib/galaxy/model/base.py index 3af9af88ebe..ca2a57207e0 100644 --- a/lib/galaxy/model/base.py +++ b/lib/galaxy/model/base.py @@ -3,6 +3,7 @@ Shared model and mapping code between Galaxy and Tool Shed, trying to generalize to generic database connections. """ import threading +from contextvars import ContextVar from inspect import ( getmembers, isclass @@ -13,19 +14,9 @@ from sqlalchemy.orm import ( scoped_session, sessionmaker ) -try: - from starlette_context import context as request_context -except ImportError: - request_context = None - from galaxy.util.bunch import Bunch - -def get_current_request(): - try: - return request_context.data['X-Request-ID'] - except RuntimeError: - return threading.get_ident() +REQUEST_ID = ContextVar('request_id', default=None) # TODO: Refactor this to be a proper class, not a bunch. @@ -35,13 +26,13 @@ class ModelMapping(Bunch): self.engine = engine SessionLocal = sessionmaker(autoflush=False, autocommit=True) versioned_session(SessionLocal) - context = scoped_session(SessionLocal, scopefunc=get_current_request if request_context is not None else None) + context = scoped_session(SessionLocal, scopefunc=self.request_scopefunc) # For backward compatibility with "context.current" # deprecated? context.current = context self._SessionLocal = SessionLocal self.session = context - self.local_session = None + self.scoped_registry = context.registry model_classes = {} for module in model_modules: @@ -54,6 +45,26 @@ class ModelMapping(Bunch): context.remove() context.configure(bind=engine) + def request_scopefunc(self): + """ + Return a value that is used as dictionary key for sqlalchemy's ScopedRegistry. + + This ensures that threads or request contexts will receive a single identical session + from the ScopedRegistry. + """ + return REQUEST_ID.get() or threading.get_ident() + + def set_request_id(self, request_id): + REQUEST_ID.set(request_id) + + def unset_request_id(self, request_id): + # Unconditionally calling self.gx_app.model.session.remove() + # would create a new session if the session was not accessed + # in a request, so we check if there is a sqlalchemy session + # for the current request in the registry. + if request_id in self.scoped_registry.registry: + self.session.remove() + @property def context(self): return self.session diff --git a/pyproject.toml b/pyproject.toml index e229236d391..3d2d6656287 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ bx-python = "*" Cheetah3 = "*" cloudauthz = "==0.6.0" cloudbridge = "*" +contextvars = {version = "*", python = "~3.6"} cwltool = "==3.0.20201109103151" dictobj = "*" docutils = "*" From 0985480b1d51284b3ff5f4238808e20e30786f46 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 20 Feb 2021 09:17:42 +0100 Subject: [PATCH 37/70] Middleware approach for setting request id --- lib/galaxy/webapps/galaxy/fast_app.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index b85c21bea56..faf4f0cb60c 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -5,6 +5,7 @@ from fastapi.responses import JSONResponse from starlette.middleware.cors import CORSMiddleware from starlette.responses import Response from starlette_context import plugins +from starlette_context import context from starlette_context.middleware import RawContextMiddleware from galaxy.exceptions import MessageException @@ -114,10 +115,30 @@ def add_galaxy_middleware(app: FastAPI, gx_app): return response +class SqlSessionPlugin(plugins.Plugin): + """ + Stores X-Request-ID set by the RequestIdPlugin in a separate contexvar + and sets and unsets the session. + """ + key = 'X-Request-ID' + + def __init__(self, *args, gx_app=None, **kwargs): + super().__init__(*args, **kwargs) + self.gx_app = gx_app + + async def process_request(self, request): + self.gx_app.model.set_request_id(context.data[self.key]) + return context.data['X-Request-ID'] + + async def enrich_response(self, arg) -> None: + self.gx_app.model.unset_request_id(context.data[self.key]) + + def initialize_fast_app(gx_webapp, gx_app): app = FastAPI( openapi_tags=api_tags_metadata ) + app.add_middleware(RawContextMiddleware, plugins=(SqlSessionPlugin(gx_app=gx_app),)) app.add_middleware(RawContextMiddleware, plugins=(plugins.RequestIdPlugin(force_new_uuid=True),)) add_exception_handler(app) add_galaxy_middleware(app, gx_app) From 871c27eed8ec720fc08444424f8a5dfe0a076c57 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 20 Feb 2021 10:26:54 +0100 Subject: [PATCH 38/70] Add test for middleware Which actually fails ... We need to ensure we don't create an unbounded amount of sessions in the session registry. The SqlSessionPlugin is supposed to handle this, but it doesn't run on unhandled exceptions, which are caught by starlette's exception handler, at which point the context is lost. --- test/unit/api/__init__.py | 0 test/unit/api/test_fastapi.py | 67 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 test/unit/api/__init__.py create mode 100644 test/unit/api/test_fastapi.py diff --git a/test/unit/api/__init__.py b/test/unit/api/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/api/test_fastapi.py b/test/unit/api/test_fastapi.py new file mode 100644 index 00000000000..2e714f1317a --- /dev/null +++ b/test/unit/api/test_fastapi.py @@ -0,0 +1,67 @@ +import uuid + +import pytest +from fastapi import FastAPI +from fastapi.param_functions import Depends +from fastapi.testclient import TestClient + + +from galaxy.webapps.galaxy.fast_app import ( + add_exception_handler, + add_sa_session_middleware, +) + +from ..unittest_utils.galaxy_mock import MockApp + +app = FastAPI() +GX_APP = None + + +def is_valid_uuid(val): + try: + uuid.UUID(str(val)) + return True + except ValueError: + return False + + +class UnexpectedException(Exception): + pass + +def get_app(): + global GX_APP + if GX_APP is None: + GX_APP = MockApp() + return GX_APP + + +@app.get("/") +async def read_main(app=Depends(get_app)): + assert app.model.scoped_registry.registry == {} + app.model.session() + assert len(app.model.scoped_registry.registry) == 1 + assert is_valid_uuid(next(iter(app.model.scoped_registry.registry.keys()))) + return {"msg": "Hello World"} + + +@app.get('/internal_server_error') +def error(app=Depends(get_app)): + assert app.model.scoped_registry.registry == {} + app.model.session() + assert len(app.model.scoped_registry.registry) == 1 + assert is_valid_uuid(next(iter(app.model.scoped_registry.registry.keys()))) + raise UnexpectedException('Oh noes!') + + +def test_sa_session_middleware(): + gx_app = get_app() + add_exception_handler(app) + add_sa_session_middleware(app, gx_app=gx_app) + client = TestClient(app) + response = client.get("/") + assert response.status_code == 200 + assert response.json() == {"msg": "Hello World"} + assert gx_app.model.scoped_registry.registry == {} + with pytest.raises(UnexpectedException): + response = client.get("/internal_server_error") + assert gx_app.model.scoped_registry.registry == {} \ No newline at end of file From 2545965425f007cfcf725be8286dc06bf9c1995d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 20 Feb 2021 11:11:09 +0100 Subject: [PATCH 39/70] Replace starlette-context with BaseHTTPMiddleware this is still not ideal, as the entire response will be loaded into memory. This is a problem with streaming responses. https://github.com/encode/starlette/issues/1012#issuecomment-673461832 is actually enlightening here: > "This means this class will either load the entirety of streaming requests into memory (this issue) and run the background before returning the response (#919 ), or if we fix those problems, that it will then encourage users to leave resources in a pending or open state, an arguably worse result. In short, it's problematic." Which is exactly what we'd be doing with a middleware, keeping resources open for longer than necessary, which we need to avoid if we ever want to run background / async tasks. I think the solution here what we already had, path operation dependencies. --- lib/galaxy/dependencies/dev-requirements.txt | 1 - .../dependencies/pinned-requirements.txt | 1 - lib/galaxy/model/base.py | 1 + lib/galaxy/webapps/galaxy/fast_app.py | 37 +++++++++---------- pyproject.toml | 1 - test/unit/api/test_fastapi.py | 4 +- 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 12308a24d3c..69e6931ad9c 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -209,7 +209,6 @@ sqlalchemy==1.3.22; (python_version >= "2.7" and python_full_version < "3.0.0") sqlitedict==1.7.0 sqlparse==0.4.1; python_version >= "3.5" starlette==0.13.6; python_version >= "3.6" and python_version < "4.0" -starlette-context==0.3.1; python_version >= "3.7" stevedore==3.3.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 0efe6d663d8..eb5e72defbd 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -169,7 +169,6 @@ sqlalchemy==1.3.22; (python_version >= "2.7" and python_full_version < "3.0.0") sqlitedict==1.7.0 sqlparse==0.4.1; python_version >= "3.5" starlette==0.13.6; python_version >= "3.6" and python_version < "4.0" -starlette-context==0.3.1; python_version >= "3.7" stevedore==3.3.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 diff --git a/lib/galaxy/model/base.py b/lib/galaxy/model/base.py index ca2a57207e0..fddc4b26695 100644 --- a/lib/galaxy/model/base.py +++ b/lib/galaxy/model/base.py @@ -14,6 +14,7 @@ from sqlalchemy.orm import ( scoped_session, sessionmaker ) + from galaxy.util.bunch import Bunch REQUEST_ID = ContextVar('request_id', default=None) diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index faf4f0cb60c..9b10de28ee9 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -1,12 +1,12 @@ +from uuid import uuid4 + from fastapi import FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.responses import JSONResponse +from starlette.middleware.base import BaseHTTPMiddleware from starlette.middleware.cors import CORSMiddleware from starlette.responses import Response -from starlette_context import plugins -from starlette_context import context -from starlette_context.middleware import RawContextMiddleware from galaxy.exceptions import MessageException from galaxy.web.framework.base import walk_controller_modules @@ -115,31 +115,30 @@ def add_galaxy_middleware(app: FastAPI, gx_app): return response -class SqlSessionPlugin(plugins.Plugin): - """ - Stores X-Request-ID set by the RequestIdPlugin in a separate contexvar - and sets and unsets the session. - """ - key = 'X-Request-ID' +def add_sa_session_middleware(app: FastAPI, gx_app): + class CustomHeaderMiddleware(BaseHTTPMiddleware): - def __init__(self, *args, gx_app=None, **kwargs): - super().__init__(*args, **kwargs) - self.gx_app = gx_app + def __init__(self, app, gx_app) -> None: + super().__init__(app) + self.gx_app = gx_app - async def process_request(self, request): - self.gx_app.model.set_request_id(context.data[self.key]) - return context.data['X-Request-ID'] + async def dispatch(self, request, call_next): + request_id = uuid4().hex + self.gx_app.model.set_request_id(request_id) + try: + response = await call_next(request) + finally: + self.gx_app.model.unset_request_id(request_id) + return response - async def enrich_response(self, arg) -> None: - self.gx_app.model.unset_request_id(context.data[self.key]) + app.add_middleware(CustomHeaderMiddleware, gx_app=gx_app) def initialize_fast_app(gx_webapp, gx_app): app = FastAPI( openapi_tags=api_tags_metadata ) - app.add_middleware(RawContextMiddleware, plugins=(SqlSessionPlugin(gx_app=gx_app),)) - app.add_middleware(RawContextMiddleware, plugins=(plugins.RequestIdPlugin(force_new_uuid=True),)) + add_sa_session_middleware(app, gx_app) add_exception_handler(app) add_galaxy_middleware(app, gx_app) wsgi_handler = WSGIMiddleware(gx_webapp) diff --git a/pyproject.toml b/pyproject.toml index 3d2d6656287..f4c0c9593eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,6 @@ sqlalchemy-migrate = "*" SQLAlchemy-Utils = "!=0.36.8" # https://github.com/kvesteri/sqlalchemy-utils/issues/462 sqlitedict = "*" sqlparse = "*" -starlette-context = "*" svgwrite = "*" uvicorn = "*" WebOb = "*" diff --git a/test/unit/api/test_fastapi.py b/test/unit/api/test_fastapi.py index 2e714f1317a..c032f7e8ad5 100644 --- a/test/unit/api/test_fastapi.py +++ b/test/unit/api/test_fastapi.py @@ -10,7 +10,6 @@ from galaxy.webapps.galaxy.fast_app import ( add_exception_handler, add_sa_session_middleware, ) - from ..unittest_utils.galaxy_mock import MockApp app = FastAPI() @@ -28,6 +27,7 @@ def is_valid_uuid(val): class UnexpectedException(Exception): pass + def get_app(): global GX_APP if GX_APP is None: @@ -64,4 +64,4 @@ def test_sa_session_middleware(): assert gx_app.model.scoped_registry.registry == {} with pytest.raises(UnexpectedException): response = client.get("/internal_server_error") - assert gx_app.model.scoped_registry.registry == {} \ No newline at end of file + assert gx_app.model.scoped_registry.registry == {} From 2d6f479132b8595709745df3f9b2473d1c211b5e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 20 Feb 2021 13:27:23 +0100 Subject: [PATCH 40/70] Move request session scoping to path dependency This avoids holding on to sessions while a response is being streamed. Also adds unit tests that verify sessions are really request or thread local (in case of background threads / non request actions (e.g job / workflow handlers)). --- lib/galaxy/dependencies/dev-requirements.txt | 3 + .../dependencies/pinned-requirements.txt | 1 + lib/galaxy/model/base.py | 22 ++- lib/galaxy/webapps/galaxy/api/__init__.py | 19 +- lib/galaxy/webapps/galaxy/fast_app.py | 29 +--- pyproject.toml | 3 + test/unit/api/__init__.py | 0 test/unit/api/test_fastapi.py | 67 -------- test/unit/jobs/test_job_wrapper.py | 2 +- ...test_request_scoped_sqlalchemy_sessions.py | 162 ++++++++++++++++++ 10 files changed, 212 insertions(+), 96 deletions(-) delete mode 100644 test/unit/api/__init__.py delete mode 100644 test/unit/api/test_fastapi.py create mode 100644 test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 69e6931ad9c..96af257e6d5 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -69,6 +69,7 @@ gxformat2==0.15.0 h11==0.12.0; python_version >= "3.6" h5py==3.1.0; python_version >= "3.6" httplib2==0.18.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" +httpx==0.16.1 humanfriendly==9.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" idna==2.10; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" imagesize==1.2.0; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.5" @@ -149,6 +150,7 @@ pyparsing==2.4.7; (python_version >= "2.6" and python_full_version < "3.0.0") or pyperclip==1.8.1; python_version >= "3.6" pyreadline==2.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" and sys_platform == "win32" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" and sys_platform == "win32" pysam==0.16.0.1 +pytest-asyncio==0.14.0 pytest-cov==2.11.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") pytest-html==3.1.1; python_version >= "3.6" pytest-metadata==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" @@ -209,6 +211,7 @@ sqlalchemy==1.3.22; (python_version >= "2.7" and python_full_version < "3.0.0") sqlitedict==1.7.0 sqlparse==0.4.1; python_version >= "3.5" starlette==0.13.6; python_version >= "3.6" and python_version < "4.0" +starlette-context==0.3.1; python_version >= "3.7" stevedore==3.3.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index eb5e72defbd..0efe6d663d8 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -169,6 +169,7 @@ sqlalchemy==1.3.22; (python_version >= "2.7" and python_full_version < "3.0.0") sqlitedict==1.7.0 sqlparse==0.4.1; python_version >= "3.5" starlette==0.13.6; python_version >= "3.6" and python_version < "4.0" +starlette-context==0.3.1; python_version >= "3.7" stevedore==3.3.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 diff --git a/lib/galaxy/model/base.py b/lib/galaxy/model/base.py index fddc4b26695..019325e1984 100644 --- a/lib/galaxy/model/base.py +++ b/lib/galaxy/model/base.py @@ -8,6 +8,7 @@ from inspect import ( getmembers, isclass ) +from typing import Dict from sqlalchemy import event from sqlalchemy.orm import ( @@ -17,7 +18,13 @@ from sqlalchemy.orm import ( from galaxy.util.bunch import Bunch -REQUEST_ID = ContextVar('request_id', default=None) + +# Create a ContextVar with mutable state, this allows sync tasks in the context +# of a request (which run within a threadpool) to see changes to the ContextVar +# state. See https://github.com/tiangolo/fastapi/issues/953#issuecomment-586006249 +# for details +_request_state: Dict[str, str] = {} +REQUEST_ID = ContextVar('request_id', default=_request_state.copy()) # TODO: Refactor this to be a proper class, not a bunch. @@ -53,10 +60,14 @@ class ModelMapping(Bunch): This ensures that threads or request contexts will receive a single identical session from the ScopedRegistry. """ - return REQUEST_ID.get() or threading.get_ident() + return REQUEST_ID.get().get('request') or threading.get_ident() - def set_request_id(self, request_id): - REQUEST_ID.set(request_id) + @staticmethod + def set_request_id(request_id): + # Set REQUEST_ID to a new dict. + # This new ContextVar value will only be seen by the current asyncio context + # and descendant threadpools, but not other threads or asyncio contexts. + return REQUEST_ID.set({'request': request_id}) def unset_request_id(self, request_id): # Unconditionally calling self.gx_app.model.session.remove() @@ -64,7 +75,8 @@ class ModelMapping(Bunch): # in a request, so we check if there is a sqlalchemy session # for the current request in the registry. if request_id in self.scoped_registry.registry: - self.session.remove() + self.scoped_registry.registry[request_id].close() + del self.scoped_registry.registry[request_id] @property def context(self): diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 479a316f136..e96ee88169a 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -2,6 +2,7 @@ This module *does not* contain API routes. It exclusively contains dependencies to be used in FastAPI routes """ from typing import ( + AsyncGenerator, cast, Optional, ) @@ -13,6 +14,10 @@ from fastapi import ( Query, ) from sqlalchemy.orm import Session +try: + from starlette_context import context as request_context +except ImportError: + request_context = None from galaxy import ( app as galaxy_app, @@ -34,8 +39,18 @@ from galaxy.web.framework.decorators import require_admin_message from galaxy.work.context import SessionRequestContext -def get_app() -> UniverseApplication: - return cast(UniverseApplication, galaxy_app.app) +async def _get_app() -> AsyncGenerator[UniverseApplication, None]: + app = cast(UniverseApplication, galaxy_app.app) + request_id = request_context.data['X-Request-ID'] + app.model.set_request_id(request_id) + try: + yield app + finally: + app.model.unset_request_id(request_id) + + +async def get_app(app=Depends(_get_app)) -> UniverseApplication: + return app def get_id_encoding_helper(app: UniverseApplication = Depends(get_app)) -> IdEncodingHelper: diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index 9b10de28ee9..c6fd1b8d2bf 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -1,12 +1,14 @@ -from uuid import uuid4 - from fastapi import FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.responses import JSONResponse -from starlette.middleware.base import BaseHTTPMiddleware from starlette.middleware.cors import CORSMiddleware from starlette.responses import Response +try: + from starlette_context.middleware import RawContextMiddleware + from starlette_context.plugins import RequestIdPlugin +except ImportError: + pass from galaxy.exceptions import MessageException from galaxy.web.framework.base import walk_controller_modules @@ -115,32 +117,17 @@ def add_galaxy_middleware(app: FastAPI, gx_app): return response -def add_sa_session_middleware(app: FastAPI, gx_app): - class CustomHeaderMiddleware(BaseHTTPMiddleware): - - def __init__(self, app, gx_app) -> None: - super().__init__(app) - self.gx_app = gx_app - - async def dispatch(self, request, call_next): - request_id = uuid4().hex - self.gx_app.model.set_request_id(request_id) - try: - response = await call_next(request) - finally: - self.gx_app.model.unset_request_id(request_id) - return response - - app.add_middleware(CustomHeaderMiddleware, gx_app=gx_app) +def add_request_id_middleware(app: FastAPI): + app.add_middleware(RawContextMiddleware, plugins=(RequestIdPlugin(force_new_uuid=True),)) def initialize_fast_app(gx_webapp, gx_app): app = FastAPI( openapi_tags=api_tags_metadata ) - add_sa_session_middleware(app, gx_app) add_exception_handler(app) add_galaxy_middleware(app, gx_app) + add_request_id_middleware(app) wsgi_handler = WSGIMiddleware(gx_webapp) for _, module in walk_controller_modules('galaxy.webapps.galaxy.api'): router = getattr(module, "router", None) diff --git a/pyproject.toml b/pyproject.toml index f4c0c9593eb..31b78e5261f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ sqlalchemy-migrate = "*" SQLAlchemy-Utils = "!=0.36.8" # https://github.com/kvesteri/sqlalchemy-utils/issues/462 sqlitedict = "*" sqlparse = "*" +starlette-context = "*" svgwrite = "*" uvicorn = "*" WebOb = "*" @@ -77,12 +78,14 @@ Whoosh = "*" zipstream-new = "*" [tool.poetry.dev-dependencies] +httpx = "*" gunicorn = "*" lxml = "!=4.2.2" markdown-it-reporter = "*" NoseHTML = "*" pygithub = "*" pytest = "*" +pytest-asyncio = "*" pytest-cov = "*" pytest-html = "*" python-irodsclient = "*" diff --git a/test/unit/api/__init__.py b/test/unit/api/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/test/unit/api/test_fastapi.py b/test/unit/api/test_fastapi.py deleted file mode 100644 index c032f7e8ad5..00000000000 --- a/test/unit/api/test_fastapi.py +++ /dev/null @@ -1,67 +0,0 @@ -import uuid - -import pytest -from fastapi import FastAPI -from fastapi.param_functions import Depends -from fastapi.testclient import TestClient - - -from galaxy.webapps.galaxy.fast_app import ( - add_exception_handler, - add_sa_session_middleware, -) -from ..unittest_utils.galaxy_mock import MockApp - -app = FastAPI() -GX_APP = None - - -def is_valid_uuid(val): - try: - uuid.UUID(str(val)) - return True - except ValueError: - return False - - -class UnexpectedException(Exception): - pass - - -def get_app(): - global GX_APP - if GX_APP is None: - GX_APP = MockApp() - return GX_APP - - -@app.get("/") -async def read_main(app=Depends(get_app)): - assert app.model.scoped_registry.registry == {} - app.model.session() - assert len(app.model.scoped_registry.registry) == 1 - assert is_valid_uuid(next(iter(app.model.scoped_registry.registry.keys()))) - return {"msg": "Hello World"} - - -@app.get('/internal_server_error') -def error(app=Depends(get_app)): - assert app.model.scoped_registry.registry == {} - app.model.session() - assert len(app.model.scoped_registry.registry) == 1 - assert is_valid_uuid(next(iter(app.model.scoped_registry.registry.keys()))) - raise UnexpectedException('Oh noes!') - - -def test_sa_session_middleware(): - gx_app = get_app() - add_exception_handler(app) - add_sa_session_middleware(app, gx_app=gx_app) - client = TestClient(app) - response = client.get("/") - assert response.status_code == 200 - assert response.json() == {"msg": "Hello World"} - assert gx_app.model.scoped_registry.registry == {} - with pytest.raises(UnexpectedException): - response = client.get("/internal_server_error") - assert gx_app.model.scoped_registry.registry == {} diff --git a/test/unit/jobs/test_job_wrapper.py b/test/unit/jobs/test_job_wrapper.py index c5e3b5e6e20..1e3a8796f3e 100644 --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -31,7 +31,7 @@ class BaseWrapperTestCase(UsesApp): job.user = User() job.object_store_id = "foo" self.model_objects = {Job: {345: job}} - self.app.model._session = MockContext(self.model_objects) + self.app.model.session = MockContext(self.model_objects) self.app.toolbox = MockToolbox(MockTool(self)) self.working_directory = os.path.join(self.test_directory, "working") diff --git a/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py b/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py new file mode 100644 index 00000000000..2a3dd5e22d7 --- /dev/null +++ b/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py @@ -0,0 +1,162 @@ +import asyncio +import concurrent.futures +import functools +import threading +import time +import uuid + +import pytest +from fastapi import FastAPI +from fastapi.param_functions import Depends +from httpx import AsyncClient +pytest.importorskip("starlette_context") +from starlette_context import context as request_context + + +from galaxy.webapps.galaxy.fast_app import add_request_id_middleware +from ..unittest_utils.galaxy_mock import MockApp + +app = FastAPI() +GX_APP = None + + +def is_valid_uuid(val): + try: + uuid.UUID(str(val)) + return True + except ValueError: + return False + + +class UnexpectedException(Exception): + pass + + +async def _get_app(): + global GX_APP + if not GX_APP: + GX_APP = MockApp() + GX_APP.stop = False + app = GX_APP + request_id = request_context.data['X-Request-ID'] + app.model.set_request_id(request_id) + try: + yield app + finally: + app.model.unset_request_id(request_id) + + +async def get_app(app=Depends(_get_app)): + return app + + +@app.get("/") +async def read_main(app=Depends(get_app)): + assert app.model.scoped_registry.registry == {} + app.model.session() + assert len(app.model.scoped_registry.registry) == 1 + request_id = app.model.request_scopefunc() + assert is_valid_uuid(request_id) + return {"msg": "Hello World"} + + +@app.get('/internal_server_error') +def error(app=Depends(get_app)): + assert app.model.scoped_registry.registry == {} + app.model.session() + assert len(app.model.scoped_registry.registry) == 1 + request_id = app.model.request_scopefunc() + assert is_valid_uuid(request_id) + raise UnexpectedException('Oh noes!') + + +@app.get('/sync_wait') +def sync_wait(app=Depends(get_app)): + app.model.session() + time.sleep(0.2) + request_id = app.model.request_scopefunc() + assert is_valid_uuid(request_id) + return request_id + + +@app.get('/async_wait') +async def async_wait(app=Depends(get_app)): + app.model.session() + await asyncio.sleep(0.2) + request_id = app.model.request_scopefunc() + assert is_valid_uuid(request_id) + return request_id + + +def assert_scoped_session_is_thread_local(gx_app): + while not gx_app.stop: + time.sleep(0.1) + request_id = gx_app.model.request_scopefunc() + assert request_id == threading.get_ident() + + +@pytest.mark.asyncio +async def test_request_scoped_sa_session_single_request(): + add_request_id_middleware(app) + async with AsyncClient(app=app, base_url="http://test") as client: + response = await client.get("/") + assert response.status_code == 200 + assert response.json() == {"msg": "Hello World"} + assert GX_APP.model.scoped_registry.registry == {} + + +@pytest.mark.asyncio +async def test_request_scoped_sa_session_exception(): + add_request_id_middleware(app) + async with AsyncClient(app=app, base_url="http://test") as client: + with pytest.raises(UnexpectedException): + await client.get("/internal_server_error") + assert GX_APP.model.scoped_registry.registry == {} + + +@pytest.mark.asyncio +async def test_request_scoped_sa_session_concurrent_requests_sync(): + add_request_id_middleware(app) + async with AsyncClient(app=app, base_url="http://test") as client: + awaitables = (client.get('/sync_wait') for _ in range(10)) + result = await asyncio.gather(*awaitables) + uuids = [] + for r in result: + assert r.status_code == 200 + uuids.append(r.json()) + assert len(set(uuids)) == 10 + assert GX_APP.model.scoped_registry.registry == {} + + +@pytest.mark.asyncio +async def test_request_scoped_sa_session_concurrent_requests_async(): + add_request_id_middleware(app) + async with AsyncClient(app=app, base_url="http://test") as client: + awaitables = (client.get('/async_wait') for _ in range(10)) + result = await asyncio.gather(*awaitables) + uuids = [] + for r in result: + assert r.status_code == 200 + uuids.append(r.json()) + assert len(set(uuids)) == 10 + assert GX_APP.model.scoped_registry.registry == {} + + +@pytest.mark.asyncio +async def test_request_scoped_sa_session_concurrent_requests_and_background_thread(): + add_request_id_middleware(app) + loop = asyncio.get_running_loop() + target = functools.partial(assert_scoped_session_is_thread_local, GX_APP) + with concurrent.futures.ThreadPoolExecutor() as pool: + background_pool = loop.run_in_executor(pool, target) + async with AsyncClient(app=app, base_url="http://test") as client: + awaitables = (client.get('/async_wait') for _ in range(10)) + result = await asyncio.gather(*awaitables) + uuids = [] + for r in result: + assert r.status_code == 200 + uuids.append(r.json()) + assert len(set(uuids)) == 10 + assert GX_APP.model.scoped_registry.registry == {} + GX_APP.stop = True + await background_pool From 61df313fb6a4baceb8e321e44ec683d054ee0761 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 21 Feb 2021 21:20:01 +0100 Subject: [PATCH 41/70] Add contextvars to data package requirement --- packages/data/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index ae42366d5fe..07977a0ec54 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -3,6 +3,7 @@ galaxy-sequence-utils galaxy-util[template] bdbag bx-python +contextvars; python_version >= "3.6" and python_version < "3.7" h5py isa-rwval parsley From a20919edaa1bad07a44e413327cf94ee19b1b4b9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 22 Feb 2021 11:12:36 +0100 Subject: [PATCH 42/70] Make hash selection mandatory --- tools/filters/secure_hash_message_digest.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/filters/secure_hash_message_digest.xml b/tools/filters/secure_hash_message_digest.xml index 50554c7181d..329f3635351 100644 --- a/tools/filters/secure_hash_message_digest.xml +++ b/tools/filters/secure_hash_message_digest.xml @@ -13,14 +13,13 @@ - + From 95859a02bed256304dc70758467904e6b0f9971a Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 22 Feb 2021 17:46:27 +0100 Subject: [PATCH 43/70] add UpgradeWorkflowAction button to UI --- client/src/components/Workflow/Editor/Index.vue | 4 ++++ client/src/components/Workflow/Editor/Options.vue | 3 +++ 2 files changed, 7 insertions(+) diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue index 3485b653448..8162287648a 100644 --- a/client/src/components/Workflow/Editor/Index.vue +++ b/client/src/components/Workflow/Editor/Index.vue @@ -109,6 +109,7 @@ @onEdit="onEdit" @onAttributes="onAttributes" @onLint="onLint" + @onUpgrade="onUpgrade" />
@@ -422,6 +423,9 @@ export default { this.$refs.lint.refresh(); showLint(); }, + onUpgrade() { + this.attemptRefactor([{'action_type': 'upgrade_workflow'}]); + }, onEdit() { this.isCanvas = true; }, diff --git a/client/src/components/Workflow/Editor/Options.vue b/client/src/components/Workflow/Editor/Options.vue index 930445279d7..06d4d4c5c2f 100644 --- a/client/src/components/Workflow/Editor/Options.vue +++ b/client/src/components/Workflow/Editor/Options.vue @@ -60,6 +60,9 @@ Auto Layout + Upgrade All Workflow Steps Download From be81d2ec098092035cbba9aa411dc704d3ace024 Mon Sep 17 00:00:00 2001 From: Mason Houtz Date: Sun, 21 Feb 2021 12:25:13 -0800 Subject: [PATCH 44/70] fixed styleguide --- .gitignore | 2 +- client/docs/root.js | 15 ++ client/docs/sections.js | 124 +++++++++++++ .../component-design/avoid-using-Galaxy.md | 14 ++ .../src/component-design/javascript-noob.md | 14 ++ .../src/component-design/never-use-jquery.md | 14 ++ .../providers-and-renderers.md | 164 ++++++++++++++++++ .../src/component-design/testing-advice.md | 3 + .../component-design/vue-component-tips.md | 82 +++++++++ client/docs/{ => src/styles}/bootstrap.md | 0 .../styles/galaxy-styles/buttons.md} | 0 .../styles/galaxy-styles/forms.md} | 0 .../styles/galaxy-styles/masthead.md} | 0 .../styles/galaxy-styles/menus.md} | 0 .../styles/galaxy-styles/messages.md} | 0 .../styles/galaxy-styles/panels.md} | 0 .../styles/galaxy-styles/tables.md} | 0 .../styles/galaxy-styles/tabs.md} | 0 client/docs/test/sample-docs/a-file.md | 1 + client/docs/test/sample-docs/another-file.md | 1 + .../contains-ommitted-file/ignored-file.md | 1 + .../contains-ommitted-file/some-loose-file.md | 1 + .../docs/test/sample-docs/has-readme/loose.md | 1 + .../test/sample-docs/has-readme/readme.md | 1 + client/docs/test/sample-docs/ignored/foo.md | 1 + .../another-nested-folder/should-exist.md | 0 .../no-summary-file/another-loose-file.md | 1 + .../test/sample-docs/no-summary-file/loose.md | 1 + client/docs/test/sections.test.js | 57 ++++++ client/src/components/History/readme.md | 69 ++++++++ client/styleguide.config.js | 91 +++++----- client/tests/jest/jest.config.js | 2 +- 32 files changed, 612 insertions(+), 48 deletions(-) create mode 100644 client/docs/root.js create mode 100644 client/docs/sections.js create mode 100644 client/docs/src/component-design/avoid-using-Galaxy.md create mode 100644 client/docs/src/component-design/javascript-noob.md create mode 100644 client/docs/src/component-design/never-use-jquery.md create mode 100644 client/docs/src/component-design/providers-and-renderers.md create mode 100644 client/docs/src/component-design/testing-advice.md create mode 100644 client/docs/src/component-design/vue-component-tips.md rename client/docs/{ => src/styles}/bootstrap.md (100%) rename client/docs/{galaxy-buttons.md => src/styles/galaxy-styles/buttons.md} (100%) rename client/docs/{galaxy-forms.md => src/styles/galaxy-styles/forms.md} (100%) rename client/docs/{galaxy-masthead.md => src/styles/galaxy-styles/masthead.md} (100%) rename client/docs/{galaxy-menus.md => src/styles/galaxy-styles/menus.md} (100%) rename client/docs/{galaxy-messages.md => src/styles/galaxy-styles/messages.md} (100%) rename client/docs/{galaxy-panels.md => src/styles/galaxy-styles/panels.md} (100%) rename client/docs/{galaxy-tables.md => src/styles/galaxy-styles/tables.md} (100%) rename client/docs/{galaxy-tabs.md => src/styles/galaxy-styles/tabs.md} (100%) create mode 100644 client/docs/test/sample-docs/a-file.md create mode 100644 client/docs/test/sample-docs/another-file.md create mode 100644 client/docs/test/sample-docs/contains-ommitted-file/ignored-file.md create mode 100644 client/docs/test/sample-docs/contains-ommitted-file/some-loose-file.md create mode 100644 client/docs/test/sample-docs/has-readme/loose.md create mode 100644 client/docs/test/sample-docs/has-readme/readme.md create mode 100644 client/docs/test/sample-docs/ignored/foo.md create mode 100644 client/docs/test/sample-docs/nested-folders/another-nested-folder/should-exist.md create mode 100644 client/docs/test/sample-docs/no-summary-file/another-loose-file.md create mode 100644 client/docs/test/sample-docs/no-summary-file/loose.md create mode 100644 client/docs/test/sections.test.js create mode 100644 client/src/components/History/readme.md diff --git a/.gitignore b/.gitignore index fc8712d025e..2e21c6be624 100644 --- a/.gitignore +++ b/.gitignore @@ -165,7 +165,7 @@ doc/build doc/schema.md doc/source/admin/config_logging_default_yaml.rst doc/source/dev/schema.md -client/styleguide +client/styleguide/dist # webpack stats client/webpack-stats.json diff --git a/client/docs/root.js b/client/docs/root.js new file mode 100644 index 00000000000..7e02f3f2ecc --- /dev/null +++ b/client/docs/root.js @@ -0,0 +1,15 @@ +/** + * Component root for rendering inside the styleguide, injects vuex store and other common elements + * into each component example section. + */ + +import store from "../src/store"; + +export default (previewComponent) => { + return { + store, + render(h) { + return h(previewComponent); + }, + }; +}; diff --git a/client/docs/sections.js b/client/docs/sections.js new file mode 100644 index 00000000000..30437cfb3a3 --- /dev/null +++ b/client/docs/sections.js @@ -0,0 +1,124 @@ +/** + * Functions for generating the sections for the style guide. Has the old style doc glob and a + * recursive tree-walker that looks through the components folder and tries to arrange found + * markdown docs in a tree. + */ + +const path = require("path"); +const glob = require("glob"); +const { humanize, titleize } = require("underscore.string"); + +/** + * Gets the table of contents sections for documentation in the components folder. + * + * @param {string} rootPath absolute path to components folder + * @return {Object} Rootnode, section map, childMap + */ +function getDocSections(rootPath, options = {}) { + const { ignore = [], docSelector = "*.@(vue|md)" } = options; + + const sections = new Map(); // absolute section directory path -> section object + const childToParent = new Map(); // absolute child path -> absolute parent path + + // create root node + const rootNode = newSection(rootPath, ignore); + sections.set(rootPath, rootNode); + childToParent.set(rootPath, null); + + // const singleGlob = path.join(rootPath, "**/*.(md|js|vue)"); + const selector = path.join(rootPath, "**", docSelector); + const allFiles = glob.sync(selector, { ignore }); + + allFiles.forEach((file) => { + const p = path.parse(file); + + // intermediate dir names between this dir and root + const relPath = path.relative(rootPath, p.dir); + const midFolders = relPath.split(path.sep); + + // build intermediate parent sections in the event that this is deeply-nested + while (midFolders.length) { + const sectionPath = path.join(rootPath, ...midFolders); + if (!sections.has(sectionPath)) { + // create new section + const section = newSection(sectionPath, ignore); + sections.set(sectionPath, section); + // register relationship for tree-build later + const parentPath = path.join(sectionPath, ".."); + childToParent.set(sectionPath, parentPath); + } + midFolders.pop(); + } + + // add file to section now that it exists + const section = sections.get(p.dir); + if (section && file !== section.content) { + section.children.add({ + name: titleize(humanize(p.name)), + content: file, + }); + } + }); + + // assemble recursive section tree under rootNode + buildSectionTree(sections, childToParent); + + // rootNode is the main result, returning the maps so the user has the option to manipulate the + // tree before handing it over to the styleguide configs + return { sections, childToParent, rootNode }; +} + +/** + * Creates a new section in the TOC from passed path. + * + * @param {string} dir absolute directory path + * @return {Object} section object + */ +function newSection(dir, ignore = []) { + const section = { + name: titleize(humanize(path.basename(dir))), + components: () => glob.sync(path.join(dir, "*.vue"), { ignore }), + children: new Set(), + sectionDepth: 1, + }; + + // summary doc is readme/docs/index.md + const summarySelector = path.join(dir, "@(readme|docs|index).md"); + const summaryDocs = glob.sync(summarySelector, { ignore, nocase: true }); + if (summaryDocs.length) { + section.content = summaryDocs[0]; + } + + return section; +} + +/** + * Turns the pair of maps into a nested object for use in the styleguide config script. Operates on + * sections array by reference. + * + * @param {Map} sections Map of path -> section object + * @param {Map} childToParent Map of childPath -> parentPath + */ +function buildSectionTree(sections, childToParent) { + // add each child to its parent + for (const [childPath, parentPath] of childToParent) { + const child = sections.get(childPath); + const parent = sections.get(parentPath); + if (child && parent && parent.children) { + parent.children.add(child); + } + } + // convert child Sets to arrays now that de-dupe not required + for (const section of sections.values()) { + section.sections = Array.from(section.children).sort((a, b) => a.name.localeCompare(b.name)); + delete section.children; + } +} + +module.exports = { + // public function + getDocSections, + // exporting these in case user wants to tweak the result tree and rebuild it + buildSectionTree, + newSection, +}; diff --git a/client/docs/src/component-design/avoid-using-Galaxy.md b/client/docs/src/component-design/avoid-using-Galaxy.md new file mode 100644 index 00000000000..15b38adc5d2 --- /dev/null +++ b/client/docs/src/component-design/avoid-using-Galaxy.md @@ -0,0 +1,14 @@ +Don't directly reference window.Galaxy in Vue components + +You've got components. Components take props. Please pass in any values you might need from +window.Galaxy as props and avoid referencing global Galaxy inside your components. I've even created +basic providers which give you access to the Galaxy.config, current user, and current user +histories. Please use them to retrieve your values, and bypass importing Galaxy altogether. + +There are use-cases where the Backbone models update over time and we need to update some value +inside Vue. Let me help you solve those problems instead of importing backbone models into Vue +components. Usually the answer is a backbone event listener that updates some relevant Vuex store. + +When writing a new component, your goal should be to replace old Galaxy functionality, not to +repackage it in another format so we continue to have Galaxy's inherent problems, the most important +of which is... \ No newline at end of file diff --git a/client/docs/src/component-design/javascript-noob.md b/client/docs/src/component-design/javascript-noob.md new file mode 100644 index 00000000000..6da166521a7 --- /dev/null +++ b/client/docs/src/component-design/javascript-noob.md @@ -0,0 +1,14 @@ +### Classes exist but they aren't as important as you may be used to. + +You may come from a class-based programming background and think that your first step should be to +make a class hierchy that does the thing you want. It doesn't help that Vue superficially looks like +a class definition, (even though what you're really doing is configuring an Observable tree). + +### Javascript is a pretty functional language. + +Javascript is largely a functional language. Its class support is limited and less useful, and +unless we switch to Typescript, we don't even have interfaces or typing, arguably the most useful +parts of a class-based language. + +In general, you will get more mileage out of javascript by embracing functional programming +approaches because that is what Javascript is good at. diff --git a/client/docs/src/component-design/never-use-jquery.md b/client/docs/src/component-design/never-use-jquery.md new file mode 100644 index 00000000000..1c89e6b9dd3 --- /dev/null +++ b/client/docs/src/component-design/never-use-jquery.md @@ -0,0 +1,14 @@ +Did you know, jQuery is old enough to drive? It's old enough to get a driver's license. jQuery is a +tool that was built to deal with inconsistencies in browsers that NO LONGER EXIST. In a couple +years, jQuery will be voting, drinking, and capable of being tried as an adult. + +If you think you need jQuery, you are mistaken. Please seek help from somebody in the wg-ui-ux +workgroup. There is nothing jQuery can provide you that isn't already part of vanilla javascript or +a standard well-tested 3rd party modern npm module. + +But that's not even the main problem. jQuery injects global initializations into every page on the +site whether you want it or not. jQuery leverages an outdated initialization paradigm which is +hugely problematic when it comes to unit testing and module building. + +One of our most important goals in redesigning Galaxy is the complete elimination of this library +from our source, along with all its invasive plugins. \ No newline at end of file diff --git a/client/docs/src/component-design/providers-and-renderers.md b/client/docs/src/component-design/providers-and-renderers.md new file mode 100644 index 00000000000..ed4064dfe6f --- /dev/null +++ b/client/docs/src/component-design/providers-and-renderers.md @@ -0,0 +1,164 @@ +We are using components in two very distinct ways. The first, "normal", kind of component will +probably look familiar to anybody whis is already passingly familiar with Vue. Here the relevant +information comes in as properties, any internal variables get defined in "data", changes go out as +events. + +## Composition Example + +```html static + + + + +``` + +In this example, we've created a component whose job is to deal with loading and updating the doodad +object. Notice that there is no markup inside the DoodadProvider other than the explicit renderless +component we previously made, but you are free to putput whatever you want in there, accessing the +doodad and saveDoddad properties as desired, as well as any other local data with the only +restriction that Vue needs a single root element in which to render. + + +## The Renderer + +```html static + + + + + +``` + +This component accepts a mandatory input object (doodad), lets the user play with a category prop, +then emits a fresh object after it's done. So what, what's the big deal? The important part to walk +away from this dumb example is the things that are NOT in this sample component. + +This component doesn't save the data. This component doesn't make ajax calls, and this component +doesn't mutate its props. What it does do is to allow the user to edit some object named "doodad" +and emits a new fresh version of that doodad when it's done. (Note also that we are using the +update:propname event syntax whenever possible [to facilitate .sync +binds](https://vuejs.org/v2/guide/components-custom-events.html#sync-Modifier)). + +Whatever happens to that new object is somebody else's job. As soon as you tie the data management +to the rendering, the re-usability of your components craters. + + +## The Provider + +As the opposite of the rendering component, a provider or renderless component, is pure lobic. It +should not know or care what your renderer is going to do with the data it provides. It is simply a +fancy way of configuring some data manipulation methods. This is one of the many ways of reusing +functionality available in Vue. Some others are [Mixins](https://vuejs.org/v2/guide/mixins.html), +[Provide/Inject](https://v3.vuejs.org/guide/component-provide-inject.html) and (in Vue3) [the +composition API](https://v3.vuejs.org/guide/composition-api-introduction.html). + + +```js static +// DoodadProvider.js + +import { loadDoodad, saveDoodad } from "./someAjaxQueryModule"; + +export default { + data() { + return { + loaded: false, + doodad: null + } + }, + methods: { + async saveDoodad(newVal) { + this.loaded = false; + const newVal = await saveDoodad(newVal); + this.doodad = newVal; + this.loaded = true; + }, + async loadDoodad(newVal) { + this.loaded = false; + const newVal = await loadDoodad(newVal); + this.loaded = true; + return newVal; + } + }, + async created() { + this.doodad = await loadDoodad(); + }, + render() { + return return this.$scopedSlots.default({ + loaded: this.loaded, + doodad: this.doodad, + saveDoodad: this.saveDoodad + }); + } +} +``` + +Here is an example of a simple possible renderless provider. The important part about this is the +render() function which simply renders [one big default +slot](https://vuejs.org/v2/guide/components-slots.html) and binds some of its own properties to that +slot for use by downstream components. + +## Unit Testing a renderless component + +Testing of a component like our editor is pretty straightforward and follows the standard Vue +guidelines. If you did a clean-enough job you won't even need to mock anything since all your data +dependencies should be delivered via props. + +But it's not so obvious how to unit-test a renderless provider. What do you check for? There's no +mandatory markup, just one big empty slot. + +```js static +// Testing a renderless component + +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue, waitForLifecyleEvent } from "jest/helpers"; +import DoodadProvider from "./DoodadProvider"; + +describe("A renderless component", () => { + const localVue = getLocalVue(); + let wrapper; + let slotProps; + + beforeEach(async () => { + wrapper = shallowMount(DoodadProvider, { + localVue, + // The mount fn allows you to hook into a slot for this very reason + scopedSlots: { + default(props) { + slotProps = props; + }, + }, + }); + + // waits for "updated" Vue lifecycle hook to fire on the renderless + // component. This is often good enough for waiting for + // an initial ajax load to finish, for example + await waitForLifecyleEvent(wrapper.vm, "updated"); + }) + + test("someProp", () => { + const { someProp } = slotProps; + expect(someProp).toExist(); + // ...more tests + }) +}) +``` \ No newline at end of file diff --git a/client/docs/src/component-design/testing-advice.md b/client/docs/src/component-design/testing-advice.md new file mode 100644 index 00000000000..8f9d1f2e08b --- /dev/null +++ b/client/docs/src/component-design/testing-advice.md @@ -0,0 +1,3 @@ +## Wrap access to browser-native resources in functions + +Unit testing in a modern javascript application runs in node without a browser. \ No newline at end of file diff --git a/client/docs/src/component-design/vue-component-tips.md b/client/docs/src/component-design/vue-component-tips.md new file mode 100644 index 00000000000..c90ff309aaf --- /dev/null +++ b/client/docs/src/component-design/vue-component-tips.md @@ -0,0 +1,82 @@ +## Understand that a component is really just a fancy function + +I'm not talking about how webpack turns it into a rendering function. That's obvious. + +I mean conceptually, props come in (like arguments) and events go out (like the return statements). +A component is a fancy kind of function that can keep emitting results and accept changing inputs +over time. In truth it more closely resembles an Observable, but an observable is ALSO a slightly +fancier kind of function. + +If you just think of a component as thing that takes inputs and emits outputs you're well on your +way to using them properly. + +The worst components are ones that might as well just be a big single script that runs. That's zero +percent better than the spaghetti we're working so hard to replace. That's just repackaging all the +problems of the old imperative class-based legacy code. + + +## Javascript is a pretty functional language. Classes exist but they aren't as important as you may be used to. + +You may come from a class-based programming background and think that your first step should be to +make a class hierchy that does the thing you want. It doesn't help that Vue superficially looks like +a class definition, (even though what you're really doing is configuring an Observable tree). + +But javascript is largely a functional language. Its class support is limited and less useful, and +unless we switch to Typescript, we don't even have interfaces or typing, arguably the most useful +parts of a class-based language. + +In general, you will get more mileage out of javascript by embracing functional programming +approaches because that is what Javascript is good at. + + +## Get comfortable with events, limit your dependence on Vuex + +New vue programmers are ok at handing props to components, but they rarely use events effectively +(at first). As a result they end up using a lot of global state, a million little data props and +relying on imperfect globalized tools like Vuex or other imported dependencies for every little +variable. + +Vuex has its uses, but not as many as you might think. Data persistence should be something that +happens near the top of your component tree, not down in the guts. + +Your first thought with a component should be: "How can I offload the handling of the results of +this component to my caller?" The answer is usually going to be events. A component that simply +accepts props and emits events can be re-used in more contexts than one that relies on an external +global state to operate. + +### Read up on .sync and v-model + +They're just fancy shorthands for a prop / event handler combination. They are fundamentally no +different from props and events, but the syntax is important to understand. + + + +## Think carefully about what should really be in "data". + +Most of good component design boils down to answering the following question: What do I want to put +in data, computed, and props? + +Data is the place where temp data goes that is not sensible to persist in Vuex or other +application-wide global state, usually because its use is very specific to the operation of this +particular component. There really should only be a few variables in data. + +### Break down your own internal dependencies + +Most components only need one or two variables in data. If you take the time to analyze your own +internal dependency tree, you will probably find that almost everything can be written in terms of +computed transformations on a small number of data and propertie + +If you have more than a few variables you (a) aren't leveraging computeds and properties, or (b) are +trying to implement too many features for just one component. It is important separate your concerns +in components just like you do in any other kind of programming. + + + +## Have a plan + +Create a design plan and an abstraction for the way the guts of your component work. Think about the +way data passes from parent components to children and back again. Break your component into +sub-components just like you would break a class into sub-methods, the same exact principles apply. + +Don't just dump a pile of spaghetti into a component, that is no better than the legacy code we are +replacing. \ No newline at end of file diff --git a/client/docs/bootstrap.md b/client/docs/src/styles/bootstrap.md similarity index 100% rename from client/docs/bootstrap.md rename to client/docs/src/styles/bootstrap.md diff --git a/client/docs/galaxy-buttons.md b/client/docs/src/styles/galaxy-styles/buttons.md similarity index 100% rename from client/docs/galaxy-buttons.md rename to client/docs/src/styles/galaxy-styles/buttons.md diff --git a/client/docs/galaxy-forms.md b/client/docs/src/styles/galaxy-styles/forms.md similarity index 100% rename from client/docs/galaxy-forms.md rename to client/docs/src/styles/galaxy-styles/forms.md diff --git a/client/docs/galaxy-masthead.md b/client/docs/src/styles/galaxy-styles/masthead.md similarity index 100% rename from client/docs/galaxy-masthead.md rename to client/docs/src/styles/galaxy-styles/masthead.md diff --git a/client/docs/galaxy-menus.md b/client/docs/src/styles/galaxy-styles/menus.md similarity index 100% rename from client/docs/galaxy-menus.md rename to client/docs/src/styles/galaxy-styles/menus.md diff --git a/client/docs/galaxy-messages.md b/client/docs/src/styles/galaxy-styles/messages.md similarity index 100% rename from client/docs/galaxy-messages.md rename to client/docs/src/styles/galaxy-styles/messages.md diff --git a/client/docs/galaxy-panels.md b/client/docs/src/styles/galaxy-styles/panels.md similarity index 100% rename from client/docs/galaxy-panels.md rename to client/docs/src/styles/galaxy-styles/panels.md diff --git a/client/docs/galaxy-tables.md b/client/docs/src/styles/galaxy-styles/tables.md similarity index 100% rename from client/docs/galaxy-tables.md rename to client/docs/src/styles/galaxy-styles/tables.md diff --git a/client/docs/galaxy-tabs.md b/client/docs/src/styles/galaxy-styles/tabs.md similarity index 100% rename from client/docs/galaxy-tabs.md rename to client/docs/src/styles/galaxy-styles/tabs.md diff --git a/client/docs/test/sample-docs/a-file.md b/client/docs/test/sample-docs/a-file.md new file mode 100644 index 00000000000..bd8d6998af4 --- /dev/null +++ b/client/docs/test/sample-docs/a-file.md @@ -0,0 +1 @@ +# testdocs/a-file.md \ No newline at end of file diff --git a/client/docs/test/sample-docs/another-file.md b/client/docs/test/sample-docs/another-file.md new file mode 100644 index 00000000000..c8d2be4519d --- /dev/null +++ b/client/docs/test/sample-docs/another-file.md @@ -0,0 +1 @@ +# testdocs/another-file.md \ No newline at end of file diff --git a/client/docs/test/sample-docs/contains-ommitted-file/ignored-file.md b/client/docs/test/sample-docs/contains-ommitted-file/ignored-file.md new file mode 100644 index 00000000000..18c16545f98 --- /dev/null +++ b/client/docs/test/sample-docs/contains-ommitted-file/ignored-file.md @@ -0,0 +1 @@ +# ignored-file.md \ No newline at end of file diff --git a/client/docs/test/sample-docs/contains-ommitted-file/some-loose-file.md b/client/docs/test/sample-docs/contains-ommitted-file/some-loose-file.md new file mode 100644 index 00000000000..7cc5029a826 --- /dev/null +++ b/client/docs/test/sample-docs/contains-ommitted-file/some-loose-file.md @@ -0,0 +1 @@ +# some-loose-file.md \ No newline at end of file diff --git a/client/docs/test/sample-docs/has-readme/loose.md b/client/docs/test/sample-docs/has-readme/loose.md new file mode 100644 index 00000000000..70ef94ec877 --- /dev/null +++ b/client/docs/test/sample-docs/has-readme/loose.md @@ -0,0 +1 @@ +# loose file \ No newline at end of file diff --git a/client/docs/test/sample-docs/has-readme/readme.md b/client/docs/test/sample-docs/has-readme/readme.md new file mode 100644 index 00000000000..eb8b03babff --- /dev/null +++ b/client/docs/test/sample-docs/has-readme/readme.md @@ -0,0 +1 @@ +# testdocs/abc/readme.md \ No newline at end of file diff --git a/client/docs/test/sample-docs/ignored/foo.md b/client/docs/test/sample-docs/ignored/foo.md new file mode 100644 index 00000000000..ad96c729e02 --- /dev/null +++ b/client/docs/test/sample-docs/ignored/foo.md @@ -0,0 +1 @@ +# foo \ No newline at end of file diff --git a/client/docs/test/sample-docs/nested-folders/another-nested-folder/should-exist.md b/client/docs/test/sample-docs/nested-folders/another-nested-folder/should-exist.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/client/docs/test/sample-docs/no-summary-file/another-loose-file.md b/client/docs/test/sample-docs/no-summary-file/another-loose-file.md new file mode 100644 index 00000000000..5be0105b76e --- /dev/null +++ b/client/docs/test/sample-docs/no-summary-file/another-loose-file.md @@ -0,0 +1 @@ +# another loose file \ No newline at end of file diff --git a/client/docs/test/sample-docs/no-summary-file/loose.md b/client/docs/test/sample-docs/no-summary-file/loose.md new file mode 100644 index 00000000000..8306ab6295b --- /dev/null +++ b/client/docs/test/sample-docs/no-summary-file/loose.md @@ -0,0 +1 @@ +# Loose file \ No newline at end of file diff --git a/client/docs/test/sections.test.js b/client/docs/test/sections.test.js new file mode 100644 index 00000000000..8b57cab5dcb --- /dev/null +++ b/client/docs/test/sections.test.js @@ -0,0 +1,57 @@ +/** + * Tests the functions which build the sections for the styleguide. + */ + +const path = require("path"); +const { getDocSections } = require("../sections"); + +const getSectionByName = (sections, name) => sections.find((o) => o.name == name); + +describe("getDocSections", () => { + const ignore = [ + // ignoring a whole directory + "**/ignored/*", + // ignoring a file + "**/ignore*", + ]; + const testDocRoot = path.join(__dirname, "sample-docs"); + const { rootNode } = getDocSections(testDocRoot, { ignore }); + + test("section generation", () => { + expect(rootNode.name).toEqual("Sample Docs"); + expect(rootNode.sections.length).toEqual(6); + expect(rootNode.content).toBeUndefined(); + }); + + test("nested subsection should appear even if no docs in intermediate folders", () => { + const deepSection = getSectionByName(rootNode.sections, "Nested Folders"); + expect(deepSection.sections.length).toEqual(1); + }); + + test("subdirectory with readme should register as summary", () => { + const subsection = getSectionByName(rootNode.sections, "Has Readme"); + + // readme interpreted as content file and not as loose section + // one other loose file + expect(subsection.content).toContain("readme.md"); + expect(subsection.sections.length).toEqual(1); + }); + + test("subdirectory with no readme file", () => { + const subsection = getSectionByName(rootNode.sections, "No Summary File"); + + // should just see 2 folders no summary + expect(subsection.content).toBeUndefined(); + expect(subsection.sections.length).toEqual(2); + }); + + test("subdirectory with ignored file", () => { + const subsection = getSectionByName(rootNode.sections, "Contains Ommitted File"); + expect(subsection.sections.length).toEqual(1); + }); + + test("ignored subdirectory", () => { + const ignoredSection = getSectionByName(rootNode.sections, "Ignored"); + expect(ignoredSection).toBeUndefined(); + }); +}); diff --git a/client/src/components/History/readme.md b/client/src/components/History/readme.md new file mode 100644 index 00000000000..ed74cef8944 --- /dev/null +++ b/client/src/components/History/readme.md @@ -0,0 +1,69 @@ +### Beta History Panel Component Tree +This is not intended to be a complete listing, but a general idea of how the components are intended +to interact with each other. + +```html static + + + + + + + + + + + + + + + + + + + + + + + + () + + + + + + + + + + + + + + + +
+ + + + + () + + + + + + + + + + +``` \ No newline at end of file diff --git a/client/styleguide.config.js b/client/styleguide.config.js index e8120e367ec..3ecd66dd6c9 100644 --- a/client/styleguide.config.js +++ b/client/styleguide.config.js @@ -1,58 +1,57 @@ const path = require("path"); -const glob = require("glob"); -const fs = require("fs"); -const merge = require("webpack-merge"); -const baseConfig = require("./webpack.config.js"); +const { getDocSections } = require("./docs/sections"); +const buildWebpack = require("./webpack.config.js"); -const webpackConfig = baseConfig(); +function getWebpack() { + const cfg = buildWebpack(); -const fileLoaderTest = /\.(png|jpg|jpeg|gif|svg|woff|woff2|ttf|eot)(\?.*$|$)/; + // looks like our src plays with the webpack publicPath dynamically, + // presumably to allow for dyamic loads, but this is a problem when + // you're not outputting code to a non-standard location. + // allowing this to happen breaks the styleguide. + cfg.module.rules.push({ + test: /onload\/publicPath/, + use: { loader: "ignore-loader" }, + }); -const fileLoaderConfigRule = { rules: [{ test: fileLoaderTest, use: ["file-loader"] }] }; + return cfg; +} -webpackConfig.module = merge.smart(webpackConfig.module, fileLoaderConfigRule); -webpackConfig.output.publicPath = ""; +// TODO: Fix broken module imports before attempting to view in styleguidef +const problemChildren = ["**/HistoryView.vue", "**/admin/DataManager/*", "**/LibraryFolder/*"]; +if (problemChildren.length) { + console.warn("Not rendering styleguide for the following components:", problemChildren); +} -webpackConfig.resolve.modules.push(path.join(__dirname, "src/style/scss")); +function getSections() { + // Style sections + const docRootPath = path.join(__dirname, "docs/src"); + const { rootNode: docRoot } = getDocSections(docRootPath, { docSelector: "*.md" }); + const [design, styles] = docRoot.sections; + delete docRoot.components; -const galaxyStyleDocs = []; -glob.sync("./docs/galaxy-*.md").forEach((file) => { - const name = file.match(/galaxy-(\w+).md/)[1]; - galaxyStyleDocs.push({ name: name, content: file }); -}); + // recursive component tree docs + const cmpPath = path.join(__dirname, "src/components"); + const { rootNode: componentDocs } = getDocSections(cmpPath, { ignore: problemChildren }); -const sections = [ - { - name: "Galaxy styles", - sections: galaxyStyleDocs, - }, - { - name: "Basic Bootstrap Styles", - content: "./docs/bootstrap.md", - }, - { - name: "Components", - // Components that are directories will get their own section - sections: glob - .sync("./src/components/*") - .map((file) => { - if (fs.lstatSync(file).isDirectory()) { - return { - name: path.basename(file), - components: file + "/**/*.vue", - }; - } - }) - .filter((v) => v), - // ...while top level components are handled here. - components: "./src/components/*.vue", - }, -]; + return [design, componentDocs, styles]; +} module.exports = { - webpackConfig, + webpackConfig: getWebpack(), + title: "Galaxy Client Resources", + sections: getSections(), + getExampleFilename(componentPath) { + return componentPath.replace(/\.(vue|js)?$/, ".md"); + }, + require: [ + "./src/style/scss/base.scss", + "./src/polyfills.js", + // "./src/bundleEntries.js" + ], + tocMode: "collapse", + renderRootJsx: "./docs/root", + styleguideDir: "./docs/dist", pagePerSection: true, - sections, - require: ["./src/style/scss/base.scss", "./src/polyfills.js", "./src/bundleEntries.js"], - vuex: "./src/store/index.js", + ignore: problemChildren, }; diff --git a/client/tests/jest/jest.config.js b/client/tests/jest/jest.config.js index 27c222d4c00..50077a062e8 100644 --- a/client/tests/jest/jest.config.js +++ b/client/tests/jest/jest.config.js @@ -118,7 +118,7 @@ module.exports = { rootDir: path.join(__dirname, "../../"), // A list of paths to directories that Jest should use to search for files in - roots: ["/src", "/tests/jest/standalone/"], + roots: ["/src", "/tests/jest/standalone/", "/docs/"], // Allows you to use a custom runner instead of Jest's default test runner // runner: "jest-runner", From a715ff5c2a620c6af3622c0f31d40037e2ca00d4 Mon Sep 17 00:00:00 2001 From: Mason Houtz Date: Sun, 21 Feb 2021 12:28:05 -0800 Subject: [PATCH 45/70] fixed gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2e21c6be624..705e1f09b01 100644 --- a/.gitignore +++ b/.gitignore @@ -165,7 +165,7 @@ doc/build doc/schema.md doc/source/admin/config_logging_default_yaml.rst doc/source/dev/schema.md -client/styleguide/dist +client/docs/dist # webpack stats client/webpack-stats.json From b9533e887482a091841bc0b20443e543ec0603f2 Mon Sep 17 00:00:00 2001 From: Mason Houtz Date: Sun, 21 Feb 2021 14:14:10 -0800 Subject: [PATCH 46/70] updating docs --- client/README.md | 135 ------------------ .../component-design/avoid-using-Galaxy.md | 14 -- .../avoid-using-global-Galaxy.md | 77 ++++++++++ .../src/component-design/javascript-noob.md | 14 -- .../src/component-design/never-use-jquery.md | 6 +- .../providers-and-renderers.md | 2 +- client/docs/src/component-design/readme.md | 71 +++++++++ .../src/component-design/testing-advice.md | 3 - .../component-design/unit-testing/readme.md | 20 +++ .../unit-testing/strategies.md | 42 ++++++ .../unit-testing/writing-tests.md | 73 ++++++++++ .../component-design/vue-component-tips.md | 82 ----------- client/styleguide.config.js | 2 +- 13 files changed, 290 insertions(+), 251 deletions(-) delete mode 100644 client/docs/src/component-design/avoid-using-Galaxy.md create mode 100644 client/docs/src/component-design/avoid-using-global-Galaxy.md delete mode 100644 client/docs/src/component-design/javascript-noob.md create mode 100644 client/docs/src/component-design/readme.md delete mode 100644 client/docs/src/component-design/testing-advice.md create mode 100644 client/docs/src/component-design/unit-testing/readme.md create mode 100644 client/docs/src/component-design/unit-testing/strategies.md create mode 100644 client/docs/src/component-design/unit-testing/writing-tests.md delete mode 100644 client/docs/src/component-design/vue-component-tips.md diff --git a/client/README.md b/client/README.md index e5bee828d32..5a90acd5109 100644 --- a/client/README.md +++ b/client/README.md @@ -164,138 +164,3 @@ terminal this starts for executing Jest tests. yarn run jest-watch Dialog yarn run jest-watch workflow/run - -### Writing a test file - -Jest will try to test any file ending in "\*.test.js". Please place your test -files inside client/src folders right next to whatever files that they are -testing. - -Jest has extensive documentation on the expect API, mocking, and more on the -[official docs page](https://jestjs.io/docs/en/getting-started.html), which will -be your best resource here. - -```javascript -// yourtestfile.test.js - -import { things } from "./yourtestfile.js"; - -describe("some module you wrote", () => { - let transientVariables; - let serviceInstances; - let testData; - - beforeEach(() => { - // setup your code (if necessary) - }); - - afterEach(() => { - // teardown your code (so it doesn't ruin the next test) - }); - - it("should do something or other", () => { - expect(workflowNodeCount()).toBe(5); - }); -}); -``` - -### Testing ~~Suggestions~~ Obligations - -#### Clearly document the intent of your test - -Please remember that these tests are not _for_ you. They're for the people who -come after you. It will be a lot easier to modify, repair and upgrade your code -if they can figure out what you were originally hoping to accomplish. Try to -use as detailed 'expect' statements as possible -- overuse of 'toBeTruthy()' -for example, can hide the intent of your test. - -Add a couple of comments. Use variable names that mean something. Nobody's -code is as self-documenting as they believe it to be. - - -#### Only test the public API that you define (carefully!) - -Internal implementations come and go with library upgrades and new tech. But -the point of the unit test is to make sure your units work as designed.... -which means you need to... you know... design your code to work in units. - -Separate your concerns and identify the developer-facing methods and functions -you expect them to use. Test THOSE. Everything else should probably be -considered an implementation detail. - -The other side of the same coin is to test *only* the unit in question. If your -component has a model that uses a service that touches Vuex, which then uses -Axios to fetch some data -- don't test all that at once. Break things apart and -mock functionality to isolate testing to units. End to end testing is a -separate thing that shouldn't be attempted using spec tests in Jest. - -Assume nobody cares _how_ your code works, we just need to know that the public -API you designed _does_ work. If performance problems or new tech necessitate a -re-write, these tests become a guide for the next implementation. - -#### Wrap native browser resources in a function so they can be easily mocked - -If your javascript needs to talk to the window object, or navigator, etc. wrap -that in a function call so that it can be easily mocked during testing. - -```javascript -// myModule.js - -// ... other code - -export function redirectTo(url) { - window.location = url; -} - -// myModule.test.js - -jest.mock("myModule", () => ({ - redirectTo: (url) => { - console.log(`I would have gone to: ${url}`); - }, -})); - -describe("some module", () => { - // .... - it("Performs a redirect when Foo is clicked"){ - expect(redirectTo.mock.calls.length).toBe(1); - } -}); -``` - -#### Implement logic in pure functions when possible - -The more of your logic that is written in deterministic functions (i.e. no -side-effects, same inputs always result in same outputs) the easier it is to -test. Just load up the functions and supply suitable test inputs. - -There is almost definitely no such thing as a well-written 1000 line function. -Most of whatever happened in that thing was probably deterministic and can be -broken up into easily testable chunks. - -### Specific test scenarios & examples - -[Mocking an imported -dependency](https://github.com/galaxyproject/galaxy/blob/dev/client/src/components/Tags/tagService.test.js) - -[Testing async -operations](https://github.com/galaxyproject/galaxy/blob/dev/client/src/components/Tags/tagService.test.js) - -[Testing a Vue component for expected rendering -output](https://github.com/galaxyproject/galaxy/blob/dev/client/src/components/Tags/StatelessTags.test.js) - -[Firing an event against a shallow mounted vue -component](https://github.com/galaxyproject/galaxy/blob/dev/client/src/components/Tags/StatelessTags.test.js) - -### The dirty secret about testing - -It's good to have tests, but testing code isn't really about testing at all, -it's actually about software design. - -Maintainable software is testable software. If you can run a unit test on your -code, then that means you must have necessarily separated your code into -testable units and you will have almost definitely written better, more -modular, more easily manipulated code, and nobody will ever contemplate using -git-blame to figure out what went wrong. - -Probably. diff --git a/client/docs/src/component-design/avoid-using-Galaxy.md b/client/docs/src/component-design/avoid-using-Galaxy.md deleted file mode 100644 index 15b38adc5d2..00000000000 --- a/client/docs/src/component-design/avoid-using-Galaxy.md +++ /dev/null @@ -1,14 +0,0 @@ -Don't directly reference window.Galaxy in Vue components - -You've got components. Components take props. Please pass in any values you might need from -window.Galaxy as props and avoid referencing global Galaxy inside your components. I've even created -basic providers which give you access to the Galaxy.config, current user, and current user -histories. Please use them to retrieve your values, and bypass importing Galaxy altogether. - -There are use-cases where the Backbone models update over time and we need to update some value -inside Vue. Let me help you solve those problems instead of importing backbone models into Vue -components. Usually the answer is a backbone event listener that updates some relevant Vuex store. - -When writing a new component, your goal should be to replace old Galaxy functionality, not to -repackage it in another format so we continue to have Galaxy's inherent problems, the most important -of which is... \ No newline at end of file diff --git a/client/docs/src/component-design/avoid-using-global-Galaxy.md b/client/docs/src/component-design/avoid-using-global-Galaxy.md new file mode 100644 index 00000000000..4a65dd9f429 --- /dev/null +++ b/client/docs/src/component-design/avoid-using-global-Galaxy.md @@ -0,0 +1,77 @@ +Without a doubt. The most problematic part of the old client is the way every important piece of +information was hung on a global variable whose initialization is completely unregulated. + +### Don't directly reference window.Galaxy in Vue components + +Components take props. Please pass in any values you might need from window.Galaxy as props and +avoid referencing global Galaxy inside your components. I've even created basic providers which give +you access to the Galaxy.config, current user, and current user histories. Please use them to +retrieve your values, and bypass importing Galaxy altogether. + +#### Sometimes you still need to update Vue from backbone as the legacy environment changes +There are definitely use-cases where the Backbone models update over time and we need to update some +value inside Vue. Let me help you solve those problems instead of importing backbone models into Vue +components. Usually the answer is a backbone event listener that updates some relevant Vuex store. + +* [Keeping Vuex in Sync with + Galaxy](https://github.com/galaxyproject/galaxy/blob/dev/client/src/store/syncVuexToGalaxy.js) + +These issues should disappear over time as the all of the old client is rebuilt in the new +ecosystem. + + +## Mount Functions + +When writing a new component, your goal should be to replace old Galaxy functionality, not to +repackage it in another format thereby persisting Galaxy's inherent problems. + +In what most people think of as a "standard" Vue application there would be only one place that Vue +is mounted to the HTML environment, and that would be in a main.j or an app.js. Most modern +single-page applications only have one starting point like that. + +However, we are incrementally replacing old Backbone views, so in its current state, Galaxy may have +several mounting functions for various components depending on where that component is intended to +fit into the existing Backbone layouts. + + +### Use the standard mount function + +A standard mount function has been provided in src/utils. This mount function accepts a component +definition, and then allows you to start up your Vue component with our standard load-out of plugins +including localization, Vuex, and a couple other utilities. This is the preferred way to mount your +component inside the old Backbone layout until the application is fully converted. + +```js static +// mountMyComponent.js + +import { mountVueComponent } from "utils/mountVueComponent"; +import MyComponent from "./MyComponent"; + +// this creates a function that will accept a props object and a DOM element on which to mount +const mounter = mountVueComponent(MyComponent); +export default mounter; +``` + +### Using the standard mount to pass in Galaxy variables as props + +```js static +import { getGalaxyInstance } from "app"; +import mountMyComponent from "./mountMyComponent"; + +const HorribleBackboneView = { + + someInitMethodYouMake() { + const Galaxy = getGalaxyInstance(); + + // pass in required props + const props = { + somePropVal: Galaxy.someDealie, + name: Galaxy.currentHistory.name, + }; + + // VM is a Vue instance. + // this.$el is some jquery selection, first item is the actual DOM object + const vm = mountMyComponent(props, this.$el[0]); + } +} +``` diff --git a/client/docs/src/component-design/javascript-noob.md b/client/docs/src/component-design/javascript-noob.md deleted file mode 100644 index 6da166521a7..00000000000 --- a/client/docs/src/component-design/javascript-noob.md +++ /dev/null @@ -1,14 +0,0 @@ -### Classes exist but they aren't as important as you may be used to. - -You may come from a class-based programming background and think that your first step should be to -make a class hierchy that does the thing you want. It doesn't help that Vue superficially looks like -a class definition, (even though what you're really doing is configuring an Observable tree). - -### Javascript is a pretty functional language. - -Javascript is largely a functional language. Its class support is limited and less useful, and -unless we switch to Typescript, we don't even have interfaces or typing, arguably the most useful -parts of a class-based language. - -In general, you will get more mileage out of javascript by embracing functional programming -approaches because that is what Javascript is good at. diff --git a/client/docs/src/component-design/never-use-jquery.md b/client/docs/src/component-design/never-use-jquery.md index 1c89e6b9dd3..c1e07181928 100644 --- a/client/docs/src/component-design/never-use-jquery.md +++ b/client/docs/src/component-design/never-use-jquery.md @@ -11,4 +11,8 @@ site whether you want it or not. jQuery leverages an outdated initialization par hugely problematic when it comes to unit testing and module building. One of our most important goals in redesigning Galaxy is the complete elimination of this library -from our source, along with all its invasive plugins. \ No newline at end of file +from our source, along with all its invasive plugins. + +### References +* [You Don't Need jQuery](https://github.com/nefe/You-Dont-Need-jQuery) +* [document.querySelector](https://developer.mozilla.org/en-US/docs/Web/API/Document/querySelector) \ No newline at end of file diff --git a/client/docs/src/component-design/providers-and-renderers.md b/client/docs/src/component-design/providers-and-renderers.md index ed4064dfe6f..03fc612fefe 100644 --- a/client/docs/src/component-design/providers-and-renderers.md +++ b/client/docs/src/component-design/providers-and-renderers.md @@ -3,7 +3,7 @@ probably look familiar to anybody whis is already passingly familiar with Vue. H information comes in as properties, any internal variables get defined in "data", changes go out as events. -## Composition Example +### Composition Example ```html static diff --git a/client/docs/src/component-design/readme.md b/client/docs/src/component-design/readme.md new file mode 100644 index 00000000000..dba0f97c1e1 --- /dev/null +++ b/client/docs/src/component-design/readme.md @@ -0,0 +1,71 @@ +### A component is really just a fancy function + +I'm not talking about how webpack turns it into a rendering function. That's obvious. + +I mean conceptually, props come in (like arguments) and events go out (like the return statements). +A component is a fancy kind of function that can keep emitting results and accept changing inputs +over time. In truth it more closely resembles an Observable, but an observable is ALSO a slightly +fancier kind of function. + +If you just think of a component as thing that takes input props and emits output events you're well +on your way to using them well. + +The worst components are ones that might as well just be a big single page script. That's zero +percent better than the spaghetti we're working so hard to replace. That's just repackaging all the +problems of the old imperative class-based legacy code. + +### Get comfortable with events, limit your dependence on Vuex + +New vue programmers are ok at handing props to components, but they rarely use events effectively +(at first). As a result they end up using a lot of global state, a million little data props and +relying on imperfect globalized tools like Vuex or other imported dependencies for every little +variable. + +Vuex definitely has its uses, but not as many as you might expect given the way it is +overly-emphasized in common tutorials. It's easy to walk away from an "Intro to Vue" video with the +idea that all data must live in Vuex all the time. That's a really undesirable situation. + +Although vuex is a well-organized (many would say over-organized) state machine, it is important +to remember that it is still a kind of global injection and deserves to be considered as such. + +* [Should I Store This Data in + Vuex](https://markus.oberlehner.net/blog/should-i-store-this-data-in-vuex/) +* [Vuex getters are great, but don’t overuse + them](https://codeburst.io/vuex-getters-are-great-but-dont-overuse-them-9c946689b414) + +Data persistence should be something that happens near the top of your component tree, not down in +the guts. + +Your first thought with a component should be: "How can I offload the handling of the results of +this component to my caller?" The answer is usually going to be events. A component that simply +accepts props and emits events can be re-used in more contexts than one that relies on an external +global state to operate. + +#### Read up on .sync and v-model + +They're just fancy shorthands for a prop / event handler combination. They are fundamentally no +different from props and events, but the syntax is important to understand. + + +### Think carefully about what should really be in "data". + +Most of good component design boils down to answering the following question: What do I want to put +in data, computed, and props? + +Data is the place where temp data goes that is not sensible to persist in Vuex or other +application-wide global state, usually because its use is very specific to the operation of this +particular component. There really should only be a few variables in data. + +#### Break down your own internal dependencies + +Most components only need one or two variables in data. If you have a large amount of data +variables, it's worth taking a little time to stop and build a mini-dependency tree for yourself. +You will probably find that almost everything can be written in terms of computed transformations on +a small number of data and properties. + +Defining your values using computes will lead to a lot less "oh I forgot to update that +variable"-style bugs. + +If you have more than a few data variables you probably (a) aren't leveraging computeds and +properties, or (b) are trying to implement too many features for just one component. It is important +separate your concerns in components just like you do in any other kind of programming. diff --git a/client/docs/src/component-design/testing-advice.md b/client/docs/src/component-design/testing-advice.md deleted file mode 100644 index 8f9d1f2e08b..00000000000 --- a/client/docs/src/component-design/testing-advice.md +++ /dev/null @@ -1,3 +0,0 @@ -## Wrap access to browser-native resources in functions - -Unit testing in a modern javascript application runs in node without a browser. \ No newline at end of file diff --git a/client/docs/src/component-design/unit-testing/readme.md b/client/docs/src/component-design/unit-testing/readme.md new file mode 100644 index 00000000000..393c1a0c117 --- /dev/null +++ b/client/docs/src/component-design/unit-testing/readme.md @@ -0,0 +1,20 @@ + +[Galaxy uses Jest](https://jestjs.io/) for its client-side unit testing +framework. + +For testing Vue components, we use the [Vue testing +utils](https://vue-test-utils.vuejs.org/) to mount individual components in a +test bed and check them for rendered features. Please use jest-based mocking +for isolating test functionality. + + +### Specific test scenarios & examples + +* [Mocking an imported +dependency](https://github.com/galaxyproject/galaxy/blob/dev/client/src/components/Tags/tagService.test.js) +* [Testing async +operations](https://github.com/galaxyproject/galaxy/blob/dev/client/src/components/Tags/tagService.test.js) +* [Testing a Vue component for expected rendering +output](https://github.com/galaxyproject/galaxy/blob/dev/client/src/components/Tags/StatelessTags.test.js) +* [Firing an event against a shallow mounted vue +component](https://github.com/galaxyproject/galaxy/blob/dev/client/src/components/Tags/StatelessTags.test.js) diff --git a/client/docs/src/component-design/unit-testing/strategies.md b/client/docs/src/component-design/unit-testing/strategies.md new file mode 100644 index 00000000000..f2c60c24783 --- /dev/null +++ b/client/docs/src/component-design/unit-testing/strategies.md @@ -0,0 +1,42 @@ +Part of making good code is making that code easy to test. + +### Implement logic in pure functions when possible + +The more of your logic that is written in deterministic functions (i.e. no +side-effects, same inputs always result in same outputs) the easier it is to +test. Just load up the functions and supply suitable test inputs. + +There is almost definitely no such thing as a well-written 1000 line function. +Most of whatever happened in that thing was probably deterministic and can be +broken up into easily testable chunks. + + +### Wrap native browser resources in a function so they can be easily mocked + +If your javascript needs to talk to the window object, or navigator, etc. wrap +that in a function call so that it can be easily mocked during testing. + +```js static +// myModule.js + +// ... other code + +export function redirectTo(url) { + window.location = url; +} + +// myModule.test.js + +jest.mock("myModule", () => ({ + redirectTo: (url) => { + console.log(`I would have gone to: ${url}`); + }, +})); + +describe("some module", () => { + // .... + it("Performs a redirect when Foo is clicked"){ + expect(redirectTo.mock.calls.length).toBe(1); + } +}); +``` \ No newline at end of file diff --git a/client/docs/src/component-design/unit-testing/writing-tests.md b/client/docs/src/component-design/unit-testing/writing-tests.md new file mode 100644 index 00000000000..8951b5797de --- /dev/null +++ b/client/docs/src/component-design/unit-testing/writing-tests.md @@ -0,0 +1,73 @@ +### Clearly document the intent of your test + +Please remember that these tests are not _for_ you. They're for the people who +come after you. It will be a lot easier to modify, repair and upgrade your code +if they can figure out what you were originally hoping to accomplish. Try to +use as detailed 'expect' statements as possible -- overuse of 'toBeTruthy()' +for example, can hide the intent of your test. + +Add a couple of comments. Use variable names that mean something. Nobody's +code is as self-documenting as they believe it to be. + + +### Only test the public API that you define + +Internal implementations come and go with library upgrades and new tech. But +the point of the unit test is to make sure your units work as designed.... +which means you need to... you know... design your code to work in units. + +Separate your concerns and identify the developer-facing methods and functions +you expect them to use. Test THOSE. Everything else should probably be +considered an implementation detail. + +The other side of the same coin is to test *only* the unit in question. If your +component has a model that uses a service that touches Vuex, which then uses +Axios to fetch some data -- don't test all that at once. Break things apart and +mock functionality to isolate testing to units. End to end testing is a +separate thing that shouldn't be attempted using spec tests in Jest. + +Assume nobody cares _how_ your code works, we just need to know that the public +API you designed _does_ work. If performance problems or new tech necessitate a +re-write, these tests become a guide for the next implementation. + + +### Writing a test file + +Jest will try to test any file ending in "\*.test.js". Please place your test +files inside client/src folders right next to whatever files that they are +testing. + +Jest has extensive documentation on the expect API, mocking, and more on the +[official docs page](https://jestjs.io/docs/en/getting-started.html), which will +be your best resource here. + +```js static +// yourcode.test.js + +import { things } from "./yourcode.js"; + +describe("some module you wrote", () => { + let transientVariables; + let serviceInstances; + let testData; + + beforeEach(() => { + // setup your code (if necessary) + }); + + afterEach(() => { + // teardown your code (so it doesn't ruin the next test) + }); + + it("should do something or other", () => { + expect(workflowNodeCount()).toBe(5); + }); +}); +``` + + +### Check out the Jest helper functions + +We have created some [common helpers for common testing +scenarios](https://github.com/galaxyproject/galaxy/blob/dev/client/tests/jest/helpers.js). + diff --git a/client/docs/src/component-design/vue-component-tips.md b/client/docs/src/component-design/vue-component-tips.md deleted file mode 100644 index c90ff309aaf..00000000000 --- a/client/docs/src/component-design/vue-component-tips.md +++ /dev/null @@ -1,82 +0,0 @@ -## Understand that a component is really just a fancy function - -I'm not talking about how webpack turns it into a rendering function. That's obvious. - -I mean conceptually, props come in (like arguments) and events go out (like the return statements). -A component is a fancy kind of function that can keep emitting results and accept changing inputs -over time. In truth it more closely resembles an Observable, but an observable is ALSO a slightly -fancier kind of function. - -If you just think of a component as thing that takes inputs and emits outputs you're well on your -way to using them properly. - -The worst components are ones that might as well just be a big single script that runs. That's zero -percent better than the spaghetti we're working so hard to replace. That's just repackaging all the -problems of the old imperative class-based legacy code. - - -## Javascript is a pretty functional language. Classes exist but they aren't as important as you may be used to. - -You may come from a class-based programming background and think that your first step should be to -make a class hierchy that does the thing you want. It doesn't help that Vue superficially looks like -a class definition, (even though what you're really doing is configuring an Observable tree). - -But javascript is largely a functional language. Its class support is limited and less useful, and -unless we switch to Typescript, we don't even have interfaces or typing, arguably the most useful -parts of a class-based language. - -In general, you will get more mileage out of javascript by embracing functional programming -approaches because that is what Javascript is good at. - - -## Get comfortable with events, limit your dependence on Vuex - -New vue programmers are ok at handing props to components, but they rarely use events effectively -(at first). As a result they end up using a lot of global state, a million little data props and -relying on imperfect globalized tools like Vuex or other imported dependencies for every little -variable. - -Vuex has its uses, but not as many as you might think. Data persistence should be something that -happens near the top of your component tree, not down in the guts. - -Your first thought with a component should be: "How can I offload the handling of the results of -this component to my caller?" The answer is usually going to be events. A component that simply -accepts props and emits events can be re-used in more contexts than one that relies on an external -global state to operate. - -### Read up on .sync and v-model - -They're just fancy shorthands for a prop / event handler combination. They are fundamentally no -different from props and events, but the syntax is important to understand. - - - -## Think carefully about what should really be in "data". - -Most of good component design boils down to answering the following question: What do I want to put -in data, computed, and props? - -Data is the place where temp data goes that is not sensible to persist in Vuex or other -application-wide global state, usually because its use is very specific to the operation of this -particular component. There really should only be a few variables in data. - -### Break down your own internal dependencies - -Most components only need one or two variables in data. If you take the time to analyze your own -internal dependency tree, you will probably find that almost everything can be written in terms of -computed transformations on a small number of data and propertie - -If you have more than a few variables you (a) aren't leveraging computeds and properties, or (b) are -trying to implement too many features for just one component. It is important separate your concerns -in components just like you do in any other kind of programming. - - - -## Have a plan - -Create a design plan and an abstraction for the way the guts of your component work. Think about the -way data passes from parent components to children and back again. Break your component into -sub-components just like you would break a class into sub-methods, the same exact principles apply. - -Don't just dump a pile of spaghetti into a component, that is no better than the legacy code we are -replacing. \ No newline at end of file diff --git a/client/styleguide.config.js b/client/styleguide.config.js index 3ecd66dd6c9..b2cc06d89bd 100644 --- a/client/styleguide.config.js +++ b/client/styleguide.config.js @@ -34,7 +34,7 @@ function getSections() { const cmpPath = path.join(__dirname, "src/components"); const { rootNode: componentDocs } = getDocSections(cmpPath, { ignore: problemChildren }); - return [design, componentDocs, styles]; + return [design, styles, componentDocs]; } module.exports = { From a52580c24655b4c13c141c625b636eec10b19eff Mon Sep 17 00:00:00 2001 From: Mason Houtz Date: Sun, 21 Feb 2021 14:38:58 -0800 Subject: [PATCH 47/70] updating unit testing docs --- .../avoid-using-global-Galaxy.md | 7 ++-- .../unit-testing/strategies.md | 35 ++++++++++++++----- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/client/docs/src/component-design/avoid-using-global-Galaxy.md b/client/docs/src/component-design/avoid-using-global-Galaxy.md index 4a65dd9f429..3e38af92fb8 100644 --- a/client/docs/src/component-design/avoid-using-global-Galaxy.md +++ b/client/docs/src/component-design/avoid-using-global-Galaxy.md @@ -10,8 +10,8 @@ retrieve your values, and bypass importing Galaxy altogether. #### Sometimes you still need to update Vue from backbone as the legacy environment changes There are definitely use-cases where the Backbone models update over time and we need to update some -value inside Vue. Let me help you solve those problems instead of importing backbone models into Vue -components. Usually the answer is a backbone event listener that updates some relevant Vuex store. +value inside Vue. Instead of importing backbone models directly into Vue components, try building a +backbone event listener that updates some relevant Vuex store. * [Keeping Vuex in Sync with Galaxy](https://github.com/galaxyproject/galaxy/blob/dev/client/src/store/syncVuexToGalaxy.js) @@ -22,9 +22,6 @@ ecosystem. ## Mount Functions -When writing a new component, your goal should be to replace old Galaxy functionality, not to -repackage it in another format thereby persisting Galaxy's inherent problems. - In what most people think of as a "standard" Vue application there would be only one place that Vue is mounted to the HTML environment, and that would be in a main.j or an app.js. Most modern single-page applications only have one starting point like that. diff --git a/client/docs/src/component-design/unit-testing/strategies.md b/client/docs/src/component-design/unit-testing/strategies.md index f2c60c24783..64c343cc850 100644 --- a/client/docs/src/component-design/unit-testing/strategies.md +++ b/client/docs/src/component-design/unit-testing/strategies.md @@ -16,6 +16,8 @@ broken up into easily testable chunks. If your javascript needs to talk to the window object, or navigator, etc. wrap that in a function call so that it can be easily mocked during testing. + +#### Your Module ```js static // myModule.js @@ -25,18 +27,33 @@ export function redirectTo(url) { window.location = url; } +export function theThingYouReallyCareAbout() { + ..... + redirectTo(somePlace); + .... +} +``` + +#### Your test file +```js static // myModule.test.js +import { theThingYouReallyCareAbout, redirectTo } from "./myModule"; -jest.mock("myModule", () => ({ - redirectTo: (url) => { - console.log(`I would have gone to: ${url}`); - }, -})); +// calling jest.mock on the appropriate import path wraps the exports and gives +// access to new testing methods like mockImplementation +jest.mock("./myModule"); +redirectTo.mockImplementation((url) => { + console.log(`I would have gone to: ${url}`); +}); -describe("some module", () => { - // .... +// You are testing theThingYouReallyCareAbout, but that function needs +// to call a window.relocate, By wrapping redirectTo as a function it was easy to mock +// and avoid errors in your test, because native browser objects do not even exist +// in the unit-testing environment. +describe("theThingYouReallyCareAbout", () => { it("Performs a redirect when Foo is clicked"){ - expect(redirectTo.mock.calls.length).toBe(1); + const result = theThingYouReallyCareAbout(abc123); + expect(result).toBe(1); } }); -``` \ No newline at end of file +``` From 596df2f53dc822825b251d22ae0c717f2d18e367 Mon Sep 17 00:00:00 2001 From: Mason Houtz Date: Sun, 21 Feb 2021 14:55:08 -0800 Subject: [PATCH 48/70] updated mount function examples --- .../avoid-using-global-Galaxy.md | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/client/docs/src/component-design/avoid-using-global-Galaxy.md b/client/docs/src/component-design/avoid-using-global-Galaxy.md index 3e38af92fb8..2a19a2597c7 100644 --- a/client/docs/src/component-design/avoid-using-global-Galaxy.md +++ b/client/docs/src/component-design/avoid-using-global-Galaxy.md @@ -31,7 +31,7 @@ several mounting functions for various components depending on where that compon fit into the existing Backbone layouts. -### Use the standard mount function +### Using the standard mount to pass in Galaxy variables as props A standard mount function has been provided in src/utils. This mount function accepts a component definition, and then allows you to start up your Vue component with our standard load-out of plugins @@ -39,36 +39,35 @@ including localization, Vuex, and a couple other utilities. This is the preferre component inside the old Backbone layout until the application is fully converted. ```js static -// mountMyComponent.js +// src/mvc/OldBackboneView.js -import { mountVueComponent } from "utils/mountVueComponent"; -import MyComponent from "./MyComponent"; - -// this creates a function that will accept a props object and a DOM element on which to mount -const mounter = mountVueComponent(MyComponent); -export default mounter; -``` - -### Using the standard mount to pass in Galaxy variables as props - -```js static import { getGalaxyInstance } from "app"; -import mountMyComponent from "./mountMyComponent"; +import MyComponent from "components/MyComponent"; +import { mountVueComponent } from "utils/mountVueComponent"; -const HorribleBackboneView = { +const OldBackboneView = { someInitMethodYouMake() { const Galaxy = getGalaxyInstance(); + const mounter = mountVueComponent(MyComponent); // pass in required props const props = { + + // Something peeled off the global galaxy somePropVal: Galaxy.someDealie, + + // ...or the current history name: Galaxy.currentHistory.name, + + // or maybe from the backbone model for this view + shoeSize: this.model.shoeSize }; // VM is a Vue instance. // this.$el is some jquery selection, first item is the actual DOM object - const vm = mountMyComponent(props, this.$el[0]); + const container = this.$el[0]; + const vm = mounter(props, container); } } ``` From d4eff9626f549f0d1cc6e96d7faf5beea8707c29 Mon Sep 17 00:00:00 2001 From: Mason Houtz Date: Mon, 22 Feb 2021 09:04:19 -0800 Subject: [PATCH 49/70] removed duplicate subsection generation --- client/docs/sections.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/docs/sections.js b/client/docs/sections.js index 30437cfb3a3..7bffe4b6cd9 100644 --- a/client/docs/sections.js +++ b/client/docs/sections.js @@ -25,7 +25,7 @@ function getDocSections(rootPath, options = {}) { sections.set(rootPath, rootNode); childToParent.set(rootPath, null); - // const singleGlob = path.join(rootPath, "**/*.(md|js|vue)"); + // get all matching doc files const selector = path.join(rootPath, "**", docSelector); const allFiles = glob.sync(selector, { ignore }); @@ -50,9 +50,9 @@ function getDocSections(rootPath, options = {}) { midFolders.pop(); } - // add file to section now that it exists + // if it's a MD file, add to parent as a subsection const section = sections.get(p.dir); - if (section && file !== section.content) { + if (section && file !== section.content && p.ext == ".md") { section.children.add({ name: titleize(humanize(p.name)), content: file, From dc961c8f69b69553c8256805c154933ea63035b2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 22 Feb 2021 18:41:35 +0100 Subject: [PATCH 50/70] Wrap core of workflow monitor thread in try/except Should prevent an exception like this: ``` galaxy.workflow.scheduling_manager DEBUG 2021-02-21 15:59:11,789 Attempting to schedule workflow invocation [(535353,)] Exception in thread WorkflowRequestMonitor.monitor_thread: Traceback (most recent call last): File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 2336, in _wrap_pool_connect return fn() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 364, in conne ct return _ConnectionFairy._checkout(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 778, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 500, in checkout rec._checkin_failed(err) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ with_traceback=exc_tb, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 497, in checkout dbapi_connection = rec.get_connection() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 610, in get_connection self.__connect() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 661, in __connect pool.logger.debug("Error on connect(): %s", e) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ with_traceback=exc_tb, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 656, in __connect connection = pool._invoke_creator(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/strategies.py", line 114, in connect return dialect.connect(*cargs, **cparams) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 493, in connect return self.dbapi.connect(*cargs, **cparams) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: FATAL: the database system is in recovery mode The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/cvmfs/main.galaxyproject.org/deps/_conda/envs/_galaxy_/lib/python3.6/threading.py", line 916, in _bootstrap_inner self.run() File "/cvmfs/main.galaxyproject.org/deps/_conda/envs/_galaxy_/lib/python3.6/threading.py", line 864, in run self._target(*self._args, **self._kwargs) File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/scheduling_manager.py", line 297, in __monitor self.__schedule(workflow_scheduler_id, workflow_scheduler) File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/scheduling_manager.py", line 305, in __schedule self.__attempt_schedule(invocation_id, workflow_scheduler) File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/scheduling_manager.py", line 311, in __attempt_schedule workflow_invocation = sa_session.query(model.WorkflowInvocation).get(invocation_id) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 1018, in get return self._get_impl(ident, loading.load_on_pk_identity) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 1135, in _get_impl return db_load_fn(self, primary_key_identity) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/loading.py", line 286, in load_on_pk_identity return q.one() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3490, in one ret = self.one_or_none() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3459, in one_or_none ret = list(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3535, in __it er__ return self._execute_and_instances(context) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3557, in _execute_and_instances querycontext, self._connection_from_session, close_with_result=True File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3572, in _get_bind_args mapper=self._bind_mapper(), clause=querycontext.statement, **kw File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3550, in _connection_from_session conn = self.session.connection(**kw) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1141, in connection execution_options=execution_options, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1150, in _connection_for_bind conn = engine._contextual_connect(**kw) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 2302, in _contextual_connect self._wrap_pool_connect(self.pool.connect, None), File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 2340, in _wrap_pool_connect e, dialect, self File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1584, in _h andle_dbapi_exception_noconnection sqlalchemy_exception, with_traceback=exc_info[2], from_=e File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 2336, in _wrap_pool_connect return fn() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 364, in connect return _ConnectionFairy._checkout(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 778, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 500, in checkout rec._checkin_failed(err) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ with_traceback=exc_tb, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 497, in check out dbapi_connection = rec.get_connection() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 610, in get_connection self.__connect() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 661, in __connect pool.logger.debug("Error on connect(): %s", e) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ with_traceback=exc_tb, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 656, in __connect connection = pool._invoke_creator(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/strategies.py", line 114, in connect return dialect.connect(*cargs, **cparams) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 493, in connect return self.dbapi.connect(*cargs, **cparams) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) FATAL: the database system is in recovery mode ``` to stop the monitor thread. --- lib/galaxy/workflow/scheduling_manager.py | 25 +++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index 46764ec3815..a51b06f95cc 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -283,19 +283,22 @@ class WorkflowRequestMonitor(Monitors): def __monitor(self): to_monitor = self.workflow_scheduling_manager.active_workflow_schedulers while self.monitor_running: - if self.invocation_grabber: - self.invocation_grabber.grab_unhandled_items() + try: + if self.invocation_grabber: + self.invocation_grabber.grab_unhandled_items() - monitor_step_timer = self.app.execution_timer_factory.get_timer( - 'internal.galaxy.workflows.scheduling_manager.monitor_step', - 'Workflow scheduling manager monitor step complete.' - ) - for workflow_scheduler_id, workflow_scheduler in to_monitor.items(): - if not self.monitor_running: - return + monitor_step_timer = self.app.execution_timer_factory.get_timer( + 'internal.galaxy.workflows.scheduling_manager.monitor_step', + 'Workflow scheduling manager monitor step complete.' + ) + for workflow_scheduler_id, workflow_scheduler in to_monitor.items(): + if not self.monitor_running: + return - self.__schedule(workflow_scheduler_id, workflow_scheduler) - log.trace(monitor_step_timer.to_str()) + self.__schedule(workflow_scheduler_id, workflow_scheduler) + log.trace(monitor_step_timer.to_str()) + except Exception: + log.exception('An exception occured scheduling while scheduling workflows') self._monitor_sleep(1) def __schedule(self, workflow_scheduler_id, workflow_scheduler): From 719f5341ad03bc5c3749aaef37cdd19d817448e7 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Mon, 22 Feb 2021 16:01:36 -0500 Subject: [PATCH 51/70] interactivetools_proxy_host typo --- lib/galaxy/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 11bde511c05..f3629feeb93 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -818,7 +818,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): # InteractiveTools propagator mapping file self.interactivetools_map = self._in_root_dir(kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite"))) self.interactivetools_prefix = kwargs.get("interactivetools_prefix", "interactivetool") - self.interactivetools_proxy_host = kwargs.get("interactivetool_proxy_host", None) + self.interactivetools_proxy_host = kwargs.get("interactivetools_proxy_host", None) self.containers_conf = parse_containers_config(self.containers_config_file) From 7181a05cb1772f51a00e330c7d15f356bff10c00 Mon Sep 17 00:00:00 2001 From: Mason Houtz Date: Mon, 22 Feb 2021 17:03:37 -0800 Subject: [PATCH 52/70] added test to avoid duplicate section if file is component example --- client/docs/sections.js | 20 ++++++++++++++++++- .../providers-and-renderers.md | 2 +- .../component-example/SomeComponent.md | 0 .../component-example/SomeComponent.vue | 3 +++ client/docs/test/sections.test.js | 7 ++++++- 5 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 client/docs/test/sample-docs/component-example/SomeComponent.md create mode 100644 client/docs/test/sample-docs/component-example/SomeComponent.vue diff --git a/client/docs/sections.js b/client/docs/sections.js index 7bffe4b6cd9..0b86475259a 100644 --- a/client/docs/sections.js +++ b/client/docs/sections.js @@ -5,6 +5,7 @@ */ const path = require("path"); +const fs = require("fs"); const glob = require("glob"); const { humanize, titleize } = require("underscore.string"); @@ -52,7 +53,7 @@ function getDocSections(rootPath, options = {}) { // if it's a MD file, add to parent as a subsection const section = sections.get(p.dir); - if (section && file !== section.content && p.ext == ".md") { + if (section && isSubsection(section, file)) { section.children.add({ name: titleize(humanize(p.name)), content: file, @@ -68,6 +69,23 @@ function getDocSections(rootPath, options = {}) { return { sections, childToParent, rootNode }; } +function isSubsection(section, file) { + const p = path.parse(file); + + // it's not a subsection if it's not a markdown file + if (p.ext !== ".md") return false; + + // it's not a subsection if it's the same file as the section content + if (file === section.content) return false; + + // it's not a subsection if it's an example for an existing component + const matchingComponentPath = path.join(p.dir, `${p.name}.vue`); + const componentExists = fs.existsSync(matchingComponentPath); + if (componentExists) return false; + + return true; +} + /** * Creates a new section in the TOC from passed path. * diff --git a/client/docs/src/component-design/providers-and-renderers.md b/client/docs/src/component-design/providers-and-renderers.md index 03fc612fefe..447e6d74b74 100644 --- a/client/docs/src/component-design/providers-and-renderers.md +++ b/client/docs/src/component-design/providers-and-renderers.md @@ -65,7 +65,7 @@ to the rendering, the re-usability of your components craters. ## The Provider -As the opposite of the rendering component, a provider or renderless component, is pure lobic. It +As the opposite of the rendering component, a provider or renderless component, is pure logic. It should not know or care what your renderer is going to do with the data it provides. It is simply a fancy way of configuring some data manipulation methods. This is one of the many ways of reusing functionality available in Vue. Some others are [Mixins](https://vuejs.org/v2/guide/mixins.html), diff --git a/client/docs/test/sample-docs/component-example/SomeComponent.md b/client/docs/test/sample-docs/component-example/SomeComponent.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/client/docs/test/sample-docs/component-example/SomeComponent.vue b/client/docs/test/sample-docs/component-example/SomeComponent.vue new file mode 100644 index 00000000000..ef519b76833 --- /dev/null +++ b/client/docs/test/sample-docs/component-example/SomeComponent.vue @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/client/docs/test/sections.test.js b/client/docs/test/sections.test.js index 8b57cab5dcb..d079ffe3162 100644 --- a/client/docs/test/sections.test.js +++ b/client/docs/test/sections.test.js @@ -19,7 +19,7 @@ describe("getDocSections", () => { test("section generation", () => { expect(rootNode.name).toEqual("Sample Docs"); - expect(rootNode.sections.length).toEqual(6); + expect(rootNode.sections.length).toEqual(7); expect(rootNode.content).toBeUndefined(); }); @@ -54,4 +54,9 @@ describe("getDocSections", () => { const ignoredSection = getSectionByName(rootNode.sections, "Ignored"); expect(ignoredSection).toBeUndefined(); }); + + test("should not create a section for a component example file", () => { + const section = getSectionByName(rootNode.sections, "Component Example"); + expect(section.sections.length).toEqual(0); + }); }); From 11c1715fc3b7b9e68417a0f258c9cc47ce1ea7a8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 22 Feb 2021 17:14:43 -0500 Subject: [PATCH 53/70] Holistic approach to dependency injection. Dependency injection is cool - it should however not be confined to the top-layer of the application. Inversion of control can help every layer and is a great way to de-couple various parts of the app. This code brings in a framework (Lagom) to build a dependency injection framework and then adds a small function (``depends``) that adapts it to FastAPI's dependency injection mechanism ("Depends"). The advantages to this approach are numerous. We don't need to write these little adapters for each component of the backend to adapt it FastAPI. For example look at the change to the roles API controller: ```diff -def get_role_manager(app: UniverseApplication = Depends(get_app)) -> RoleManager: - return app.role_manager - - @cbv(router) class FastAPIRoles: - role_manager: RoleManager = Depends(get_role_manager) + role_manager: RoleManager = depends(RoleManager) ``` This is much less boilerplate. We don't need to implement & type that function get_role_manager and we don't need to bring in the import on UniverseApplication. Additionally, we've got a clean abstraction that shields us from ``fastapi`` imports in all of our controllers. It should make it more possible to switch to new frameworks and such as the Python ecosystem matures. Also, the same DI that is used to inject ``RoleManager`` into this contoller is used to inject app into RoleManager when it is constructed during application initialization. Any component being managed by UniverseApplication can now rely on its constructor arguments to be injected if it wants. It is easy to see the cool examples on FastAPI and think it is just a technology for controllers, but it totally is not. I don't think there is really a way to use FastAPI's dependency injection outside the context of that framework, but even if one could Lagom is superior. All the auto-wiring is by type and requires zero framework and zero configuration (https://github.com/meadsteve/lagom#auto-wiring-with-zero-configuraton). Having a web framework provide these framework-bound extension points for injecting stuff into controllers was the state of the art of Java like 15 years ago. Skipping that whole learning process and using the type system and auto-wiring that isn't dependent on framework annotation really jumps out to Java circa 8 years ago! Why Lagom is an interesting question. When researching DI frameworks, I couldn't find a clear winner but Lagom focus on type annotations versus annotating by name makes it clearly more modern than a lot frameworks by much bigger names (https://github.com/meadsteve/lagom/blob/master/docs/comparison.md). The other type-centric framework that had even a nice interface that I found was punq (https://punq.readthedocs.io/en/latest/). The development just doesn't seem as active on punq. While I didn't land up using the integration Lagom seems to have async frameworks in mind (https://github.com/meadsteve/lagom/blob/master/lagom/integrations/fast_api.py), so that is another plus. Ultimately though I think I can swap between these two with like 10 lines of code switch, they do cool things with very simple interfaces and neither requires a bunch of investment in annotation on your components. --- lib/galaxy/app.py | 116 +++++++++++------- lib/galaxy/auth/__init__.py | 7 +- lib/galaxy/config_watchers.py | 3 +- lib/galaxy/dependencies/dev-requirements.txt | 1 + .../dependencies/pinned-requirements.txt | 1 + lib/galaxy/di.py | 37 ++++++ lib/galaxy/jobs/__init__.py | 12 +- lib/galaxy/jobs/handler.py | 11 +- lib/galaxy/jobs/manager.py | 7 +- lib/galaxy/managers/api_keys.py | 9 +- lib/galaxy/managers/base.py | 31 +++-- lib/galaxy/managers/citations.py | 4 +- lib/galaxy/managers/cloud.py | 3 - lib/galaxy/managers/collections.py | 39 +++--- lib/galaxy/managers/datasets.py | 9 +- lib/galaxy/managers/display_applications.py | 3 +- lib/galaxy/managers/hdas.py | 11 +- lib/galaxy/managers/hdcas.py | 15 +-- lib/galaxy/managers/histories.py | 25 ++-- lib/galaxy/managers/history_contents.py | 9 +- lib/galaxy/managers/jobs.py | 19 ++- lib/galaxy/managers/lddas.py | 3 +- lib/galaxy/managers/libraries.py | 17 +-- lib/galaxy/managers/pages.py | 34 ++--- lib/galaxy/managers/roles.py | 3 - lib/galaxy/managers/session.py | 13 +- lib/galaxy/managers/sharable.py | 3 +- lib/galaxy/managers/tags.py | 3 +- lib/galaxy/managers/tools.py | 3 - lib/galaxy/managers/users.py | 14 +-- lib/galaxy/managers/visualizations.py | 8 +- lib/galaxy/managers/workflows.py | 5 +- lib/galaxy/model/__init__.py | 1 + lib/galaxy/model/base.py | 14 ++- lib/galaxy/model/mapping.py | 8 +- lib/galaxy/model/tags.py | 8 +- lib/galaxy/quota/__init__.py | 10 +- lib/galaxy/structured_app.py | 36 ++++-- lib/galaxy/tools/__init__.py | 3 +- lib/galaxy/tools/cache.py | 16 ++- lib/galaxy/tools/data_manager/manager.py | 8 +- lib/galaxy/tools/repositories.py | 1 + lib/galaxy/visualization/genomes.py | 6 +- .../visualization/plugins/resource_parser.py | 4 +- lib/galaxy/webapps/base/controller.py | 10 +- lib/galaxy/webapps/base/webapp.py | 17 ++- lib/galaxy/webapps/galaxy/api/__init__.py | 73 +++++++---- lib/galaxy/webapps/galaxy/api/annotations.py | 14 +-- lib/galaxy/webapps/galaxy/api/authenticate.py | 4 +- lib/galaxy/webapps/galaxy/api/cloud.py | 11 +- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 7 +- .../webapps/galaxy/api/configuration.py | 41 +++---- .../galaxy/api/container_resolution.py | 7 +- .../webapps/galaxy/api/dataset_collections.py | 14 +-- lib/galaxy/webapps/galaxy/api/datasets.py | 35 +++--- lib/galaxy/webapps/galaxy/api/datatypes.py | 20 ++- .../galaxy/api/display_applications.py | 10 +- .../webapps/galaxy/api/dynamic_tools.py | 3 - .../webapps/galaxy/api/extended_metadata.py | 9 +- .../webapps/galaxy/api/folder_contents.py | 16 ++- lib/galaxy/webapps/galaxy/api/folders.py | 8 +- lib/galaxy/webapps/galaxy/api/forms.py | 6 +- lib/galaxy/webapps/galaxy/api/genomes.py | 27 ++-- lib/galaxy/webapps/galaxy/api/group_roles.py | 9 +- lib/galaxy/webapps/galaxy/api/group_users.py | 9 +- lib/galaxy/webapps/galaxy/api/groups.py | 10 +- lib/galaxy/webapps/galaxy/api/histories.py | 23 ++-- .../webapps/galaxy/api/history_contents.py | 23 ++-- lib/galaxy/webapps/galaxy/api/item_tags.py | 4 +- lib/galaxy/webapps/galaxy/api/job_files.py | 4 +- lib/galaxy/webapps/galaxy/api/job_lock.py | 13 +- lib/galaxy/webapps/galaxy/api/job_ports.py | 8 +- lib/galaxy/webapps/galaxy/api/jobs.py | 36 ++---- lib/galaxy/webapps/galaxy/api/libraries.py | 9 +- .../webapps/galaxy/api/library_contents.py | 10 +- .../webapps/galaxy/api/library_datasets.py | 7 +- lib/galaxy/webapps/galaxy/api/licenses.py | 18 ++- lib/galaxy/webapps/galaxy/api/metrics.py | 7 +- .../webapps/galaxy/api/page_revisions.py | 9 +- lib/galaxy/webapps/galaxy/api/pages.py | 32 ++--- lib/galaxy/webapps/galaxy/api/plugins.py | 11 +- lib/galaxy/webapps/galaxy/api/provenance.py | 12 +- lib/galaxy/webapps/galaxy/api/quotas.py | 5 +- lib/galaxy/webapps/galaxy/api/remote_files.py | 25 ++-- lib/galaxy/webapps/galaxy/api/roles.py | 38 +++--- lib/galaxy/webapps/galaxy/api/search.py | 8 +- lib/galaxy/webapps/galaxy/api/tags.py | 21 ++-- .../webapps/galaxy/api/tool_dependencies.py | 7 +- .../webapps/galaxy/api/tool_entry_points.py | 7 +- .../galaxy/api/tool_shed_repositories.py | 4 +- lib/galaxy/webapps/galaxy/api/tools.py | 13 +- lib/galaxy/webapps/galaxy/api/toolshed.py | 4 +- lib/galaxy/webapps/galaxy/api/tours.py | 28 ++--- lib/galaxy/webapps/galaxy/api/trs_consumer.py | 11 +- lib/galaxy/webapps/galaxy/api/trs_search.py | 12 +- lib/galaxy/webapps/galaxy/api/uploads.py | 4 +- lib/galaxy/webapps/galaxy/api/users.py | 16 +-- .../webapps/galaxy/api/visualizations.py | 11 +- lib/galaxy/webapps/galaxy/api/webhooks.py | 4 +- lib/galaxy/webapps/galaxy/api/workflows.py | 7 +- lib/galaxy/webapps/galaxy/buildapp.py | 2 +- .../webapps/galaxy/controllers/dataset.py | 13 +- .../webapps/galaxy/controllers/history.py | 12 +- .../webapps/galaxy/controllers/openid.py | 3 +- lib/galaxy/webapps/galaxy/controllers/page.py | 14 ++- lib/galaxy/webapps/galaxy/controllers/root.py | 10 +- lib/galaxy/webapps/galaxy/controllers/user.py | 6 +- .../galaxy/controllers/visualization.py | 9 +- lib/galaxy/webapps/galaxy/fast_app.py | 4 +- lib/galaxy/workflow/run_request.py | 3 +- lib/tool_shed/webapp/app.py | 44 ++++--- lib/tool_shed/webapp/model/mapping.py | 12 +- packages/app/galaxy/di.py | 1 + packages/app/requirements.txt | 1 + packages/webapps/requirements.txt | 2 + pyproject.toml | 2 + test/unit/managers/base.py | 2 +- test/unit/managers/test_CollectionManager.py | 8 +- test/unit/managers/test_DatasetManager.py | 2 +- test/unit/managers/test_HDAManager.py | 6 +- test/unit/managers/test_HDCAManager.py | 10 +- .../managers/test_HistoryContentsManager.py | 10 +- test/unit/managers/test_HistoryManager.py | 18 +-- test/unit/managers/test_TagHandler.py | 14 +-- test/unit/managers/test_UserManager.py | 21 +--- test/unit/tools_support.py | 1 + test/unit/unittest_utils/galaxy_mock.py | 20 ++- 127 files changed, 873 insertions(+), 781 deletions(-) create mode 100644 lib/galaxy/di.py create mode 120000 packages/app/galaxy/di.py diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 95d2f1b3a3a..eb69840d8a5 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -4,13 +4,18 @@ import sys import time from typing import Any +from sqlalchemy.orm.scoping import ( + scoped_session, +) + import galaxy.model import galaxy.model.security import galaxy.queues import galaxy.security -from galaxy import config, jobs +from galaxy import auth, config, jobs from galaxy.config_watchers import ConfigWatchers from galaxy.containers import build_container_interfaces +from galaxy.datatypes.registry import Registry from galaxy.files import ConfiguredFileSources from galaxy.job_metrics import JobMetrics from galaxy.managers.api_keys import ApiKeyManager @@ -19,25 +24,30 @@ from galaxy.managers.folders import FolderManager from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager from galaxy.managers.interactivetool import InteractiveToolManager +from galaxy.managers.jobs import JobSearch from galaxy.managers.libraries import LibraryManager from galaxy.managers.roles import RoleManager +from galaxy.managers.session import GalaxySessionManager from galaxy.managers.tools import DynamicToolManager from galaxy.managers.users import UserManager from galaxy.managers.workflows import ( WorkflowContentsManager, WorkflowsManager, ) +from galaxy.model.base import SharedModelMapping from galaxy.model.database_heartbeat import DatabaseHeartbeat +from galaxy.model.mapping import GalaxyModelMapping from galaxy.model.tags import GalaxyTagHandler from galaxy.queue_worker import ( GalaxyQueueWorker, send_local_control_task, ) -from galaxy.quota import get_quota_agent +from galaxy.quota import get_quota_agent, QuotaAgent +from galaxy.security.idencoding import IdEncodingHelper from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager from galaxy.tool_util.deps.views import DependencyResolversView -from galaxy.tool_util.verify import test_data +from galaxy.tool_util.verify.test_data import TestDataResolver from galaxy.tools.cache import ( ToolCache, ToolShedRepositoryCache @@ -45,7 +55,7 @@ from galaxy.tools.cache import ( from galaxy.tools.data_manager.manager import DataManagers from galaxy.tools.error_reports import ErrorReports from galaxy.tools.special_tools import load_lib_tools -from galaxy.tours import build_tours_registry +from galaxy.tours import build_tours_registry, ToursRegistry from galaxy.util import ( ExecutionTimer, heartbeat, @@ -56,19 +66,23 @@ from galaxy.visualization.genomes import Genomes from galaxy.visualization.plugins.registry import VisualizationsRegistry from galaxy.web import url_for from galaxy.web.proxy import ProxyManager -from galaxy.web_stack import application_stack_instance +from galaxy.web_stack import application_stack_instance, ApplicationStack from galaxy.webhooks import WebhooksRegistry from galaxy.workflow.trs_proxy import TrsProxy -from .structured_app import StructuredApp +from .di import Container +from .structured_app import BasicApp, StructuredApp log = logging.getLogger(__name__) app = None -class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin): +class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container): """Encapsulates the state of a Universe application""" def __init__(self, **kwargs) -> None: + super().__init__() + self._register_singleton(BasicApp, self) + self._register_singleton(StructuredApp, self) if not log.handlers: # Paste didn't handle it, so we need a temporary basic log # configured. The handler added here gets dumped and replaced with @@ -80,19 +94,19 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin): startup_timer = ExecutionTimer() self.new_installation = False # Read config file and check for errors - self.config: Any = config.Configuration(**kwargs) + self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs)) self.config.check() config.configure_logging(self.config) - self.execution_timer_factory = ExecutionTimerFactory(self.config) + self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config)) self.configure_fluent_log() # A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions - self.application_stack = application_stack_instance(app=self) + self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self)) self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self) self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs) self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config) # queue_worker *can* be initialized with a queue, but here we don't # want to and we'll allow postfork to bind and start it. - self.queue_worker = GalaxyQueueWorker(self) + self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self)) self._configure_tool_shed_registry() self._configure_object_store(fsmon=True) @@ -105,23 +119,30 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin): # Security helper self._configure_security() + self._register_singleton(IdEncodingHelper, self.security) + self._register_singleton(SharedModelMapping, self.model) + self._register_singleton(GalaxyModelMapping, self.model) + self._register_singleton(scoped_session, self.model.context) # Tag handler - self.tag_handler = GalaxyTagHandler(self.model.context) - self.dataset_collections_service = DatasetCollectionManager(self) - self.history_manager = HistoryManager(self) - self.hda_manager = HDAManager(self) - self.workflow_manager = WorkflowsManager(self) - self.workflow_contents_manager = WorkflowContentsManager(self) - self.dependency_resolvers_view = DependencyResolversView(self) - self.test_data_resolver = test_data.TestDataResolver(file_dirs=self.config.tool_test_data_directories) - self.library_folder_manager = FolderManager() - self.library_manager = LibraryManager() - self.role_manager = RoleManager(self) - self.dynamic_tool_manager = DynamicToolManager(self) - self.api_keys_manager = ApiKeyManager(app) + self.tag_handler = self._register_singleton(GalaxyTagHandler) + self.user_manager = self._register_singleton(UserManager) + self._register_singleton(GalaxySessionManager) + self.hda_manager = self._register_singleton(HDAManager) + self.history_manager = self._register_singleton(HistoryManager) + self.job_search = self._register_singleton(JobSearch) + self.dataset_collections_service = self._register_singleton(DatasetCollectionManager) + self.workflow_manager = self._register_singleton(WorkflowsManager) + self.workflow_contents_manager = self._register_singleton(WorkflowContentsManager) + self.dependency_resolvers_view = self._register_singleton(DependencyResolversView, DependencyResolversView(self)) + self.test_data_resolver = self._register_singleton(TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories)) + self.library_folder_manager = self._register_singleton(FolderManager) + self.library_manager = self._register_singleton(LibraryManager) + self.role_manager = self._register_singleton(RoleManager) + self.dynamic_tool_manager = self._register_singleton(DynamicToolManager) + self.api_keys_manager = self._register_singleton(ApiKeyManager) # ConfiguredFileSources - self.file_sources = ConfiguredFileSources.from_app_config(self.config) + self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) # Tool Data Tables self._configure_tool_data_tables(from_shed_config=False) @@ -129,36 +150,37 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin): self._configure_genome_builds(data_table_name="__dbkeys__", load_old_style=True) # Genomes - self.genomes = Genomes(self) + self.genomes = self._register_singleton(Genomes) # Data providers registry. - self.data_provider_registry = DataProviderRegistry() + self.data_provider_registry = self._register_singleton(DataProviderRegistry) # Initialize job metrics manager, needs to be in place before # config so per-destination modifications can be made. - self.job_metrics = JobMetrics(self.config.job_metrics_config_file, app=self) + self.job_metrics = self._register_singleton(JobMetrics, JobMetrics(self.config.job_metrics_config_file, app=self)) # Initialize error report plugins. - self.error_reports = ErrorReports(self.config.error_report_file, app=self) + self.error_reports = self._register_singleton(ErrorReports, ErrorReports(self.config.error_report_file, app=self)) # Initialize the job management configuration - self.job_config = jobs.JobConfiguration(self) + self.job_config = self._register_singleton(jobs.JobConfiguration) # Setup a Tool Cache - self.tool_cache = ToolCache() - self.tool_shed_repository_cache = ToolShedRepositoryCache(self) + self.tool_cache = self._register_singleton(ToolCache) + self.tool_shed_repository_cache = self._register_singleton(ToolShedRepositoryCache) # Watch various config files for immediate reload - self.watchers = ConfigWatchers(self) + self.watchers = self._register_singleton(ConfigWatchers) self._configure_tool_config_files() - self.installed_repository_manager = InstalledRepositoryManager(self) + self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self)) self._configure_datatypes_registry(self.installed_repository_manager) + self._register_singleton(Registry, self.datatypes_registry) galaxy.model.set_datatypes_registry(self.datatypes_registry) self._configure_toolbox() # Load Data Manager - self.data_managers = DataManagers(self) + self.data_managers = self._register_singleton(DataManagers) # Load the update repository manager. - self.update_repository_manager = UpdateRepositoryManager(self) + self.update_repository_manager = self._register_singleton(UpdateRepositoryManager, UpdateRepositoryManager(self)) # Load proprietary datatype converters and display applications. self.installed_repository_manager.load_proprietary_converters_and_display_applications() # Load datatype display applications defined in local datatypes_conf.xml @@ -171,26 +193,26 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin): load_lib_tools(self.toolbox) self.toolbox.persist_cache(register_postfork=True) # visualizations registry: associates resources with visualizations, controls how to render - self.visualizations_registry = VisualizationsRegistry( + self.visualizations_registry = self._register_singleton(VisualizationsRegistry, VisualizationsRegistry( self, directories_setting=self.config.visualization_plugins_directory, - template_cache_dir=self.config.template_cache_path) + template_cache_dir=self.config.template_cache_path)) # Tours registry - self.tour_registry = build_tours_registry(self.config.tour_config_dir) + tour_registry = build_tours_registry(self.config.tour_config_dir) + self.tour_registry = tour_registry + self[ToursRegistry] = tour_registry # type: ignore # Webhooks registry - self.webhooks_registry = WebhooksRegistry(self.config.webhooks_dir) + self.webhooks_registry = self._register_singleton(WebhooksRegistry, WebhooksRegistry(self.config.webhooks_dir)) # Load security policy. self.security_agent = self.model.security_agent self.host_security_agent = galaxy.model.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions) # Load quota management. - self.quota_agent = get_quota_agent(self.config, self.model) + self.quota_agent = self._register_singleton(QuotaAgent, get_quota_agent(self.config, self.model)) # Heartbeat for thread profiling self.heartbeat = None - from galaxy import auth - self.auth_manager = auth.AuthManager(self) - self.user_manager = UserManager(self) + self.auth_manager = self._register_singleton(auth.AuthManager, auth.AuthManager(self.config)) # Start the heartbeat process if configured and available (wait until # postfork if using uWSGI) if self.config.use_heartbeat: @@ -221,7 +243,7 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin): # Start the job manager from galaxy.jobs import manager - self.job_manager = manager.JobManager(self) + self.job_manager = self._register_singleton(manager.JobManager) self.application_stack.register_postfork_function(self.job_manager.start) self.proxy_manager = ProxyManager(self.config) @@ -229,7 +251,7 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin): # Must be initialized after job_config. self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self) - self.trs_proxy = TrsProxy(self.config) + self.trs_proxy = self._register_singleton(TrsProxy, TrsProxy(self.config)) # Must be initialized after any component that might make use of stack messaging is configured. Alternatively if # it becomes more commonly needed we could create a prefork function registration method like we do with # postfork functions. @@ -341,7 +363,7 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin): self.trace_logger = None @property - def is_job_handler(self): + def is_job_handler(self) -> bool: return (self.config.track_jobs_in_database and self.job_config.is_handler) or not self.config.track_jobs_in_database diff --git a/lib/galaxy/auth/__init__.py b/lib/galaxy/auth/__init__.py index 72014f8a9b8..f7976ea32a1 100644 --- a/lib/galaxy/auth/__init__.py +++ b/lib/galaxy/auth/__init__.py @@ -12,10 +12,9 @@ log = logging.getLogger(__name__) class AuthManager: - def __init__(self, app): - self.__app = app - self.redact_username_in_logs = app.config.redact_username_in_logs - self.authenticators = get_authenticators(app.config.auth_config_file, app.config.is_set('auth_config_file')) + def __init__(self, config): + self.redact_username_in_logs = config.redact_username_in_logs + self.authenticators = get_authenticators(config.auth_config_file, config.is_set('auth_config_file')) def check_registration_allowed(self, email, username, password): """Checks if the provided email/username is allowed to register.""" diff --git a/lib/galaxy/config_watchers.py b/lib/galaxy/config_watchers.py index 5ac57915231..c20ed8651e5 100644 --- a/lib/galaxy/config_watchers.py +++ b/lib/galaxy/config_watchers.py @@ -2,6 +2,7 @@ import logging from os.path import dirname from galaxy.queue_worker import job_rule_modules +from galaxy.structured_app import StructuredApp from galaxy.tools.toolbox.watcher import ( get_tool_conf_watcher, get_tool_watcher, @@ -14,7 +15,7 @@ log = logging.getLogger(__name__) class ConfigWatchers: """Contains ToolConfWatcher, ToolWatcher and ToolDataWatcher objects.""" - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app self.active = False # ToolConfWatcher objects will watch the tool_cache if the tool_cache is passed into get_tool_conf_watcher. diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 96af257e6d5..4c00a59e444 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -85,6 +85,7 @@ jsonpatch==1.28; python_version >= "3.6" and python_full_version < "3.0.0" or py jsonpointer==2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" keystoneauth1==4.3.0; python_version >= "3.6" kombu==5.0.2; python_version >= "3.6" +lagom==1.0.0; python_version >= "3.6" and python_version < "4.0" lockfile==0.12.2; python_version >= "3.6" and python_version < "4" lxml==4.6.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") mako==1.1.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 0efe6d663d8..56e67ca289d 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -75,6 +75,7 @@ jsonpatch==1.28; python_version >= "3.6" and python_full_version < "3.0.0" or py jsonpointer==2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" keystoneauth1==4.3.0; python_version >= "3.6" kombu==5.0.2; python_version >= "3.6" +lagom==1.0.0; python_version >= "3.6" and python_version < "4.0" lockfile==0.12.2; python_version >= "3.6" and python_version < "4" lxml==4.6.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") mako==1.1.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") diff --git a/lib/galaxy/di.py b/lib/galaxy/di.py new file mode 100644 index 00000000000..8736140b996 --- /dev/null +++ b/lib/galaxy/di.py @@ -0,0 +1,37 @@ +"""Dependency injection framework for Galaxy-type apps.""" +from typing import Optional, Type, TypeVar + +from lagom import Container as LagomContainer +from lagom.exceptions import UnresolvableType + +T = TypeVar("T") + + +class Container(LagomContainer): + """Abstraction around lagom to provide a dependency injection context. + + Abstractions used by Galaxy should come through this interface so we can swap + out the backend as needed. For instance https://punq.readthedocs.io/en/latest/ + containers look very nice and would allow us to also inject by name (e.g. for + config variables for instance). + """ + + def _register_singleton(self, dep_type: Type[T], instance: Optional[T] = None) -> T: + if instance is None: + # create an instance from the context and register it as a singleton + instance = self[dep_type] + self[dep_type] = instance + return self[dep_type] + + def resolve_or_none(self, dep_type: Type[T]) -> Optional[T]: + """Resolve the dependent type or just return None. + + If resolution is impossible assume caller has a backup plan for + constructing the desired object. Used to construct controllers that + may or may not be resolvable (some have upgraded but legacy framework still + works). + """ + try: + return self[dep_type] + except UnresolvableType: + return None diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index c0f0c218a6c..d9877bdaa94 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -18,6 +18,7 @@ from abc import ( abstractmethod, ) from json import loads +from typing import Any, Dict, List import packaging.version import yaml @@ -56,6 +57,7 @@ from galaxy.jobs.runners import BaseJobRunner, JobState from galaxy.metadata import get_metadata_compute_strategy from galaxy.model import store from galaxy.objectstore import ObjectStorePopulator +from galaxy.structured_app import StructuredApp from galaxy.tool_util.deps import requirements from galaxy.tool_util.output_checker import ( check_output, @@ -277,6 +279,13 @@ class JobConfiguration(ConfiguresHandlers): These features are configured in the job configuration, by default, ``job_conf.xml`` """ + runner_plugins: List[dict] + handlers: dict + handler_runner_plugins: Dict[str, str] + tools: Dict[str, list] + resource_groups: Dict[str, list] + destinations: Dict[str, tuple] + resource_parameters: Dict[str, Any] DEFAULT_BASE_HANDLER_POOLS = ('job-handlers',) DEFAULT_NWORKERS = 4 @@ -292,7 +301,7 @@ class JobConfiguration(ConfiguresHandlers): """ - def __init__(self, app): + def __init__(self, app: StructuredApp): """Parse the job configuration XML. """ self.app = app @@ -306,7 +315,6 @@ class JobConfiguration(ConfiguresHandlers): self.handler_max_grab = None self.handler_ready_window_size = None self.destinations = {} - self.destination_tags = {} self.default_destination_id = None self.tools = {} self.resource_groups = {} diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index fded5e0bdc0..fd091badcd5 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -40,7 +40,16 @@ JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_READY, JOB_DELETED, DEFAULT_JOB_PUT_FAILURE_MESSAGE = 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' -class JobHandler: +class JobHandlerI: + + def start(self): + pass + + def shutdown(self): + pass + + +class JobHandler(JobHandlerI): """ Handle the preparation, running, tracking, and finishing of jobs """ diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index 0f90e5a9e24..a9740288e75 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -9,6 +9,7 @@ from sqlalchemy.sql.expression import null from galaxy.exceptions import HandlerAssignmentError, ToolExecutionError from galaxy.jobs import handler, NoopQueue from galaxy.model import Job +from galaxy.structured_app import StructuredApp from galaxy.web_stack.message import JobHandlerMessage log = logging.getLogger(__name__) @@ -18,7 +19,9 @@ class JobManager: """ Highest level interface to job management. """ - def __init__(self, app): + job_handler: handler.JobHandlerI + + def __init__(self, app: StructuredApp): self.app = app self.job_lock = False if self.app.is_job_handler: @@ -110,7 +113,7 @@ class NoopManager: pass -class NoopHandler: +class NoopHandler(handler.JobHandlerI): """ Implements the JobHandler interface but does nothing """ diff --git a/lib/galaxy/managers/api_keys.py b/lib/galaxy/managers/api_keys.py index d816739f3e6..0617a3743fe 100644 --- a/lib/galaxy/managers/api_keys.py +++ b/lib/galaxy/managers/api_keys.py @@ -1,9 +1,12 @@ +from galaxy.structured_app import BasicApp + + class ApiKeyManager: - def __init__(self, app): + def __init__(self, app: BasicApp): self.app = app - def create_api_key(self, user): + def create_api_key(self, user) -> str: guid = self.app.security.get_new_guid() new_key = self.app.model.APIKeys() new_key.user_id = user.id @@ -13,7 +16,7 @@ class ApiKeyManager: sa_session.flush() return guid - def get_or_create_api_key(self, user): + def get_or_create_api_key(self, user) -> str: # Logic Galaxy has always used - but it would appear to have a race # condition. Worth fixing? Would kind of need a message queue to fix # in multiple process mode. diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index 2eb1fd2bbc5..de18f4c279c 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -28,14 +28,16 @@ attribute change to a model object. import datetime import logging import re -from typing import Type +from typing import Callable, Dict, List, Optional, Set, Type import routes import sqlalchemy +from sqlalchemy.orm.scoping import scoped_session from galaxy import exceptions from galaxy import model from galaxy.model import tool_shed_install +from galaxy.structured_app import BasicApp, StructuredApp from galaxy.util import namedtuple log = logging.getLogger(__name__) @@ -158,11 +160,12 @@ class ModelManager: """ model_class: type = object foreign_key_name: str + app: BasicApp - def __init__(self, app): + def __init__(self, app: BasicApp): self.app = app - def session(self): + def session(self) -> scoped_session: return self.app.model.context def _session_setattr(self, item, attr, val, fn=None, flush=True): @@ -485,7 +488,7 @@ class HasAModelManager: # examples where this doesn't really work are ConfigurationSerializer (no manager) # and contents (2 managers) - def __init__(self, app, manager=None, **kwargs): + def __init__(self, app: StructuredApp, manager=None, **kwargs): self._manager = manager @property @@ -494,7 +497,7 @@ class HasAModelManager: # PRECONDITION: assumes self.app is assigned elsewhere if not self._manager: # TODO: pass this serializer to it - self._manager = self.model_manager_class(self.app) + self._manager = self.app[self.model_manager_class] # this will error for unset model_manager_class'es return self._manager @@ -536,8 +539,10 @@ class ModelSerializer(HasAModelManager): """ #: 'service' to use for getting urls - use class var to allow overriding when testing url_for = staticmethod(routes.url_for) + default_view: Optional[str] + views: Dict[str, List[str]] - def __init__(self, app, **kwargs): + def __init__(self, app: StructuredApp, **kwargs): """ Set up serializer map, any additional serializable keys, and views here. """ @@ -548,9 +553,9 @@ class ModelSerializer(HasAModelManager): # this allows us to: 'mention' the key without adding the default serializer # TODO: we may want to eventually error if a key is requested # that is in neither serializable_keyset or serializers - self.serializable_keyset = set() + self.serializable_keyset: Set[str] = set() # a map of dictionary keys to the functions (often lambdas) that create the values for those keys - self.serializers = {} + self.serializers: Dict[str, Callable] = {} # add subclass serializers defined there self.add_serializers() # update the keyset by the serializers (removing the responsibility from subclasses) @@ -708,15 +713,15 @@ class ModelDeserializer(HasAModelManager): """ # TODO:?? a larger question is: which should be first? Deserialize then validate - or - validate then deserialize? - def __init__(self, app, validator=None, **kwargs): + def __init__(self, app: StructuredApp, validator=None, **kwargs): """ Set up deserializers and validator. """ super().__init__(app, **kwargs) self.app = app - self.deserializers = {} - self.deserializable_keyset = set() + self.deserializers: Dict[str, Callable] = {} + self.deserializable_keyset: Set[str] = set() self.add_deserializers() # a sub object that can validate incoming values self.validate = validator or ModelValidator(self.app) @@ -896,8 +901,10 @@ class ModelFilterParser(HasAModelManager): model_class: type parsed_filter = parsed_filter + orm_filter_parsers: Dict[str, Dict] + fn_filter_parsers: Dict[str, Dict] - def __init__(self, app, **kwargs): + def __init__(self, app: StructuredApp, **kwargs): """ Set up serializer map, any additional serializable keys, and views here. """ diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index c8ac75f8aab..9573c911f70 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -5,12 +5,14 @@ import requests from beaker.cache import CacheManager from beaker.util import parse_cache_config_options +from galaxy.structured_app import BasicApp + log = logging.getLogger(__name__) class CitationsManager: - def __init__(self, app): + def __init__(self, app: BasicApp) -> None: self.app = app self.doi_cache = DoiCache(app.config) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index e4cb42243e9..dd3697f6c1c 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -50,9 +50,6 @@ class CloudManager(sharable.SharableModelManager): # setting this property. model_class = model.History - def __init__(self, app, *args, **kwargs): - super().__init__(app, *args, **kwargs) - @staticmethod def configure_provider(provider, credentials): """ diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index 017da542906..c45942401e9 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -8,20 +8,21 @@ from galaxy.exceptions import ( MessageException, RequestParameterInvalidException ) -from galaxy.managers import ( - hdas, - histories, - lddas, -) from galaxy.managers.collections_util import validate_input_element_identifiers -from galaxy.model import tags from galaxy.model.dataset_collections import builder from galaxy.model.dataset_collections.matching import MatchingCollections from galaxy.model.dataset_collections.registry import DATASET_COLLECTION_TYPES_REGISTRY from galaxy.model.dataset_collections.type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY +from galaxy.model.mapping import GalaxyModelMapping +from galaxy.model.tags import GalaxyTagHandler +from galaxy.security.idencoding import IdEncodingHelper from galaxy.util import ( validation ) +from .hdas import HDAManager +from .histories import HistoryManager +from .lddas import LDDAManager + log = logging.getLogger(__name__) @@ -36,16 +37,24 @@ class DatasetCollectionManager: """ ELEMENTS_UNINITIALIZED = object() - def __init__(self, app): + def __init__( + self, + model: GalaxyModelMapping, + security: IdEncodingHelper, + hda_manager: HDAManager, + history_manager: HistoryManager, + tag_handler: GalaxyTagHandler, + ldda_manager: LDDAManager, + ): self.type_registry = DATASET_COLLECTION_TYPES_REGISTRY self.collection_type_descriptions = COLLECTION_TYPE_DESCRIPTION_FACTORY - self.model = app.model - self.security = app.security + self.model = model + self.security = security - self.hda_manager = hdas.HDAManager(app) - self.history_manager = histories.HistoryManager(app) - self.tag_handler = tags.GalaxyTagHandler(app.model.context) - self.ldda_manager = lddas.LDDAManager(app) + self.hda_manager = hda_manager + self.history_manager = history_manager + self.tag_handler = tag_handler + self.ldda_manager = ldda_manager def precreate_dataset_collection_instance(self, trans, parent, name, structure, implicit_inputs=None, implicit_output_name=None, tags=None, completed_collection=None): # TODO: prebuild all required HIDs and send them in so no need to flush in between. @@ -136,7 +145,7 @@ class DatasetCollectionManager: def _create_instance_for_collection(self, trans, parent, name, dataset_collection, implicit_output_name=None, implicit_inputs=None, tags=None, set_hid=True, flush=True): if isinstance(parent, model.History): - dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation( + dataset_collection_instance = model.HistoryDatasetCollectionAssociation( collection=dataset_collection, name=name, ) @@ -153,7 +162,7 @@ class DatasetCollectionManager: parent.add_dataset_collection(dataset_collection_instance) elif isinstance(parent, model.LibraryFolder): - dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation( + dataset_collection_instance = model.LibraryDatasetCollectionAssociation( collection=dataset_collection, folder=parent, name=name, diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 3bb6e901130..0f7092e3a5e 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -18,6 +18,7 @@ from galaxy.managers import ( secured, users ) +from galaxy.structured_app import StructuredApp from galaxy.util.checkers import check_binary log = logging.getLogger(__name__) @@ -32,7 +33,7 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl # TODO:?? get + error_if_uploading is common pattern, should upload check be worked into access/owed? - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.permissions = DatasetRBACPermissions(app) # needed for admin test @@ -142,11 +143,11 @@ class DatasetRBACPermissions: class DatasetSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): model_manager_class = DatasetManager - def __init__(self, app): + def __init__(self, app: StructuredApp, user_manager: users.UserManager): super().__init__(app) self.dataset_manager = self.manager # needed for admin test - self.user_manager = users.UserManager(app) + self.user_manager = user_manager self.default_view = 'summary' self.add_view('summary', [ @@ -438,7 +439,7 @@ class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): def __init__(self, app): - self.dataset_serializer = DatasetSerializer(app) + self.dataset_serializer = app[DatasetSerializer] super().__init__(app) def add_serializers(self): diff --git a/lib/galaxy/managers/display_applications.py b/lib/galaxy/managers/display_applications.py index 2da27193768..de88e8b9a9d 100644 --- a/lib/galaxy/managers/display_applications.py +++ b/lib/galaxy/managers/display_applications.py @@ -5,6 +5,7 @@ from typing import ( ) from galaxy.datatypes.registry import Registry +from galaxy.structured_app import StructuredApp log = logging.getLogger(__name__) @@ -12,7 +13,7 @@ log = logging.getLogger(__name__) class DisplayApplicationsManager: """Interface/service object for sharing logic between controllers.""" - def __init__(self, app): + def __init__(self, app: StructuredApp): self._app = app @property diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index f6b664fb2db..c4d4093a452 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -18,8 +18,9 @@ from galaxy.managers import ( datasets, secured, taggable, - users + users, ) +from galaxy.structured_app import StructuredApp log = logging.getLogger(__name__) @@ -40,12 +41,12 @@ class HDAManager(datasets.DatasetAssociationManager, # TODO: move what makes sense into DatasetManager # TODO: which of these are common with LDDAs and can be pushed down into DatasetAssociationManager? - def __init__(self, app): + def __init__(self, app: StructuredApp, user_manager: users.UserManager): """ Set up and initialize other managers needed by hdas. """ super().__init__(app) - self.user_manager = users.UserManager(app) + self.user_manager = user_manager def get_owned_ids(self, object_ids, history=None): """Get owned IDs. @@ -247,7 +248,7 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize annotatable.AnnotatableSerializerMixin): model_manager_class = HDAManager - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.hda_manager = self.manager @@ -506,7 +507,7 @@ class HDADeserializer(datasets.DatasetAssociationDeserializer, """ model_manager_class = HDAManager - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.hda_manager = self.manager diff --git a/lib/galaxy/managers/hdcas.py b/lib/galaxy/managers/hdcas.py index 7aa696676bd..ac25761b6d0 100644 --- a/lib/galaxy/managers/hdcas.py +++ b/lib/galaxy/managers/hdcas.py @@ -16,6 +16,7 @@ from galaxy.managers import ( taggable ) from galaxy.managers.collections_util import get_hda_and_element_identifiers +from galaxy.structured_app import StructuredApp from galaxy.util.zipstream import ZipstreamWrapper @@ -55,12 +56,6 @@ class HDCAManager( tag_assoc = model.HistoryDatasetCollectionTagAssociation annotation_assoc = model.HistoryDatasetCollectionAssociationAnnotationAssociation - def __init__(self, app): - """ - Set up and initialize other managers needed by hdcas. - """ - super().__init__(app) - def map_datasets(self, content, fn, *parents): """ Iterate over the datasets of a given collection, recursing into collections, and @@ -92,7 +87,7 @@ class DCESerializer(base.ModelSerializer): Serializer for DatasetCollectionElements. """ - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.hda_serializer = hdas.HDASerializer(app) self.dc_serializer = DCSerializer(app, dce_serializer=self) @@ -126,7 +121,7 @@ class DCSerializer(base.ModelSerializer): Serializer for DatasetCollections. """ - def __init__(self, app, dce_serializer=None): + def __init__(self, app: StructuredApp, dce_serializer=None): super().__init__(app) self.dce_serializer = dce_serializer or DCESerializer(app) @@ -165,7 +160,7 @@ class DCASerializer(base.ModelSerializer): Base (abstract) Serializer class for HDCAs and LDCAs. """ - def __init__(self, app, dce_serializer=None): + def __init__(self, app: StructuredApp, dce_serializer=None): super().__init__(app) self.dce_serializer = dce_serializer or DCESerializer(app) @@ -220,7 +215,7 @@ class HDCASerializer( Serializer for HistoryDatasetCollectionAssociations. """ - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.hdca_manager = HDCAManager(app) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 6649139a0b9..84fd2c5a121 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -21,6 +21,7 @@ from galaxy.managers import ( history_contents, sharable ) +from galaxy.structured_app import StructuredApp log = logging.getLogger(__name__) @@ -37,11 +38,11 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix # TODO: incorporate imp/exp (or alias to) - def __init__(self, app, *args, **kwargs): - super().__init__(app, *args, **kwargs) - self.hda_manager = hdas.HDAManager(app) - self.contents_manager = history_contents.HistoryContentsManager(app) - self.contents_filters = history_contents.HistoryContentsFilters(app) + def __init__(self, app: StructuredApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters): + super().__init__(app) + self.hda_manager = hda_manager + self.contents_manager = contents_manager + self.contents_filters = contents_filters def copy(self, history, user, **kwargs): """ @@ -171,7 +172,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix class HistoryExportView: - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app def get_exports(self, trans, history_id): @@ -223,13 +224,13 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri model_manager_class = HistoryManager SINGLE_CHAR_ABBR = 'h' - def __init__(self, app, **kwargs): - super().__init__(app, **kwargs) + def __init__(self, app: StructuredApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer): + super().__init__(app) self.history_manager = self.manager - self.hda_manager = hdas.HDAManager(app) - self.hda_serializer = hdas.HDASerializer(app) - self.history_contents_serializer = history_contents.HistoryContentsSerializer(app) + self.hda_manager = hda_manager + self.hda_serializer = hda_serializer + self.history_contents_serializer = history_contents_serializer self.default_view = 'summary' self.add_view('summary', [ @@ -442,7 +443,7 @@ class HistoryDeserializer(sharable.SharableModelDeserializer, deletable.Purgable """ model_manager_class = HistoryManager - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.history_manager = self.manager diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 23e2f72e477..9765a7c0a95 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -32,6 +32,7 @@ from galaxy.managers import ( taggable, tools ) +from galaxy.structured_app import StructuredApp log = logging.getLogger(__name__) @@ -72,10 +73,10 @@ class HistoryContentsManager(containers.ContainerManagerMixin): ) default_order_by = 'hid' - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app - self.contained_manager = self.contained_class_manager_class(app) - self.subcontainer_manager = self.subcontainer_class_manager_class(app) + self.contained_manager = app[self.contained_class_manager_class] + self.subcontainer_manager = app[self.subcontainer_class_manager_class] # ---- interface def contained(self, container, filters=None, limit=None, offset=None, order_by=None, **kwargs): @@ -426,7 +427,7 @@ class HistoryContentsSerializer(base.ModelSerializer, deletable.PurgableSerializ """ model_manager_class = HistoryContentsManager - def __init__(self, app, **kwargs): + def __init__(self, app: StructuredApp, **kwargs): super().__init__(app, **kwargs) self.default_view = 'summary' diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 8ebdd07400a..18385e3e76b 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -21,6 +21,7 @@ from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.datasets import DatasetManager from galaxy.managers.hdas import HDAManager from galaxy.managers.lddas import LDDAManager +from galaxy.structured_app import StructuredApp from galaxy.util import ( defaultdict, ExecutionTimer, @@ -55,11 +56,11 @@ def get_path_key(path_tuple): class JobManager: - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app self.dataset_manager = DatasetManager(app) - def job_lock(self): + def job_lock(self) -> JobLock: return JobLock(active=self.app.job_manager.job_lock) def update_job_lock(self, job_lock: JobLock): @@ -93,12 +94,18 @@ class JobManager: class JobSearch: """Search for jobs using tool inputs or other jobs""" - def __init__(self, app): + def __init__( + self, + app: StructuredApp, + hda_manager: HDAManager, + dataset_collection_manager: DatasetCollectionManager, + ldda_manager: LDDAManager + ): self.app = app self.sa_session = app.model.context - self.hda_manager = HDAManager(app) - self.dataset_collection_manager = DatasetCollectionManager(app) - self.ldda_manager = LDDAManager(app) + self.hda_manager = hda_manager + self.dataset_collection_manager = dataset_collection_manager + self.ldda_manager = ldda_manager self.decode_id = self.app.security.decode_id def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state='ok'): diff --git a/lib/galaxy/managers/lddas.py b/lib/galaxy/managers/lddas.py index 0361f6cb1ad..53d2a0a9b28 100644 --- a/lib/galaxy/managers/lddas.py +++ b/lib/galaxy/managers/lddas.py @@ -3,6 +3,7 @@ import logging from galaxy import model, util from galaxy.managers import base as manager_base from galaxy.managers.datasets import DatasetAssociationManager +from galaxy.structured_app import StructuredApp log = logging.getLogger(__name__) @@ -13,7 +14,7 @@ class LDDAManager(DatasetAssociationManager): """ model_class = model.LibraryDatasetDatasetAssociation - def __init__(self, app): + def __init__(self, app: StructuredApp): """ Set up and initialize other managers needed by lddas. """ diff --git a/lib/galaxy/managers/libraries.py b/lib/galaxy/managers/libraries.py index 4441a125915..7934ee33367 100644 --- a/lib/galaxy/managers/libraries.py +++ b/lib/galaxy/managers/libraries.py @@ -23,7 +23,6 @@ from galaxy.managers import ( ) from galaxy.managers.context import ProvidesAppContext from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import StructuredApp from galaxy.util import ( pretty_print_time_interval, unicodify, @@ -38,9 +37,6 @@ class LibraryManager: Interface/service object for interacting with libraries. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def get(self, trans, decoded_library_id, check_accessible=True): """ Get the library from the DB. @@ -329,10 +325,15 @@ class LibrariesManager: Interface/service object for sharing logic between controllers. """ - def __init__(self, app: StructuredApp): - self.folder_manager = folders.FolderManager() - self.library_manager = LibraryManager() - self.role_manager = roles.RoleManager(app) + def __init__( + self, + folder_manager: folders.FolderManager, + library_manager: LibraryManager, + role_manager: roles.RoleManager, + ): + self.folder_manager = folder_manager + self.library_manager = library_manager + self.role_manager = role_manager def index(self, trans: ProvidesAppContext, deleted: Optional[bool] = False) -> List[Any]: """Returns a list of summary data for all libraries. diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index 840d5359454..e1620ef0a15 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -11,6 +11,7 @@ from enum import Enum from html.entities import name2codepoint from html.parser import HTMLParser from typing import ( + Callable, List, Optional, ) @@ -22,15 +23,15 @@ from pydantic import ( from galaxy import exceptions, model from galaxy.managers import base, sharable -from galaxy.managers.hdas import HDAManager +from galaxy.managers.context import ProvidesHistoryContext from galaxy.managers.markdown_util import ( internal_galaxy_markdown_to_pdf, ready_galaxy_markdown_for_export, ready_galaxy_markdown_for_import, ) -from galaxy.managers.workflows import WorkflowsManager from galaxy.model.item_attrs import UsesAnnotations from galaxy.schema.fields import EncodedDatabaseIdField +from galaxy.structured_app import StructuredApp from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html @@ -179,7 +180,7 @@ class PagesManager: Common interface/service logic for interactions with pages. """ - def __init__(self, app): + def __init__(self, app: StructuredApp): self.manager = PageManager(app) def index(self, trans, deleted: bool = False) -> PageSummaryList: @@ -282,11 +283,11 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): annotation_assoc = model.PageAnnotationAssociation rating_assoc = model.PageRatingAssociation - def __init__(self, app, *args, **kwargs): + def __init__(self, app: StructuredApp): """ """ - super().__init__(app, *args, **kwargs) - self.workflow_manager = WorkflowsManager(app) + super().__init__(app) + self.workflow_manager = app.workflow_manager def create(self, trans, payload): user = trans.get_user() @@ -379,7 +380,6 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): content = ready_galaxy_markdown_for_import(trans, content) else: raise exceptions.RequestParameterInvalidException("content_format [%s] must be either html or markdown" % content_format) - return content def rewrite_content_for_export(self, trans, as_dict): @@ -406,7 +406,7 @@ class PageSerializer(sharable.SharableModelSerializer): model_manager_class = PageManager SINGLE_CHAR_ABBR = 'p' - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.page_manager = PageManager(app) @@ -427,7 +427,7 @@ class PageDeserializer(sharable.SharableModelDeserializer): """ model_manager_class = PageManager - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.page_manager = self.manager @@ -450,7 +450,7 @@ class PageContentProcessor(HTMLParser): 'source', 'track', 'wbr' } - def __init__(self, trans, render_embed_html_fn): + def __init__(self, trans, render_embed_html_fn: Callable): HTMLParser.__init__(self) self.trans = trans self.ignore_content = False @@ -627,28 +627,28 @@ PAGE_CLASS_MAPPING = { } -def placeholderRenderForEdit(trans, item_class, item_id): +def placeholderRenderForEdit(trans: ProvidesHistoryContext, item_class, item_id): return placeholderRenderForSave(trans, item_class, item_id, encode=True) -def placeholderRenderForSave(trans, item_class, item_id, encode=False): +def placeholderRenderForSave(trans: ProvidesHistoryContext, item_class, item_id, encode=False): encoded_item_id, decoded_item_id = get_page_identifiers(item_id, trans.app) item_name = '' if item_class == 'History': - history = trans.sa_session.query(trans.model.History).get(decoded_item_id) + history = trans.sa_session.query(model.History).get(decoded_item_id) history = base.security_check(trans, history, False, True) item_name = history.name elif item_class == 'HistoryDatasetAssociation': - hda = trans.sa_session.query(trans.model.HistoryDatasetAssociation).get(decoded_item_id) - hda_manager = HDAManager(trans.app) + hda = trans.sa_session.query(model.HistoryDatasetAssociation).get(decoded_item_id) + hda_manager = trans.app.hda_manager hda = hda_manager.get_accessible(decoded_item_id, trans.user) item_name = hda.name elif item_class == 'StoredWorkflow': - wf = trans.sa_session.query(trans.model.StoredWorkflow).get(decoded_item_id) + wf = trans.sa_session.query(model.StoredWorkflow).get(decoded_item_id) wf = base.security_check(trans, wf, False, True) item_name = wf.name elif item_class == 'Visualization': - visualization = trans.sa_session.query(trans.model.Visualization).get(decoded_item_id) + visualization = trans.sa_session.query(model.Visualization).get(decoded_item_id) visualization = base.security_check(trans, visualization, False, True) item_name = visualization.title class_shorthand = PAGE_CLASS_MAPPING[item_class] diff --git a/lib/galaxy/managers/roles.py b/lib/galaxy/managers/roles.py index 088e637fd7b..dae89dcf744 100644 --- a/lib/galaxy/managers/roles.py +++ b/lib/galaxy/managers/roles.py @@ -53,9 +53,6 @@ class RoleManager(base.ModelManager): user_assoc = model.UserRoleAssociation group_assoc = model.GroupRoleAssociation - def __init__(self, app): - super().__init__(app) - def get(self, trans: ProvidesUserContext, decoded_role_id): """ Method loads the role from the DB based on the given role id. diff --git a/lib/galaxy/managers/session.py b/lib/galaxy/managers/session.py index bf099e9e9ec..8ec49cd23ad 100644 --- a/lib/galaxy/managers/session.py +++ b/lib/galaxy/managers/session.py @@ -1,3 +1,5 @@ +import logging + from sqlalchemy import ( and_, true, @@ -6,16 +8,23 @@ from sqlalchemy.orm import ( joinedload, ) +from galaxy.model.base import SharedModelMapping + +log = logging.getLogger(__name__) + class GalaxySessionManager: """Manages GalaxySession.""" - def __init__(self, model): + def __init__(self, model: SharedModelMapping): self.model = model + self.sa_session = model.context def get_session_from_session_key(self, session_key: str): """Returns GalaxySession if session_key is valid.""" - galaxy_session = self.model.session.query(self.model.GalaxySession).filter( + # going through self.model since this can be used by Galaxy or Toolshed despite + # type annotations + galaxy_session = self.sa_session.query(self.model.GalaxySession).filter( and_( self.model.GalaxySession.table.c.session_key == session_key, self.model.GalaxySession.table.c.is_valid == true()) diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py index 8b60017f573..80876dad3cd 100644 --- a/lib/galaxy/managers/sharable.py +++ b/lib/galaxy/managers/sharable.py @@ -25,6 +25,7 @@ from galaxy.managers import ( users ) from galaxy.model import UserShareAssociation +from galaxy.structured_app import StructuredApp log = logging.getLogger(__name__) @@ -40,7 +41,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur #: the single character abbreviation used in username_and_slug: e.g. 'h' for histories: u/user/h/slug SINGLE_CHAR_ABBR: Optional[str] = None - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) # user manager is needed to check access/ownership/admin self.user_manager = users.UserManager(app) diff --git a/lib/galaxy/managers/tags.py b/lib/galaxy/managers/tags.py index 6f059f49014..7af0c5da1ba 100644 --- a/lib/galaxy/managers/tags.py +++ b/lib/galaxy/managers/tags.py @@ -62,6 +62,7 @@ class TagsManager: """ tag_handler = GalaxyTagHandlerSession(trans.sa_session) id = trans.security.decode_id(payload.item_id) - item_class = tag_handler.item_tag_assoc_info[payload.item_class].item_class + item_class_name = str(payload.item_class) + item_class = tag_handler.item_tag_assoc_info[item_class_name].item_class item = trans.sa_session.query(item_class).filter(item_class.id == id).first() return item diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index 932021ff318..c320a726445 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -17,9 +17,6 @@ class DynamicToolManager(ModelManager): """ model_class = model.DynamicTool - def __init__(self, app): - super().__init__(app) - def get_tool_by_uuid(self, uuid): dynamic_tool = self._one_or_none( self.query().filter(self.model_class.uuid == uuid) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 9a76959b626..e8b0493bc31 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -30,6 +30,7 @@ from galaxy.security.validate_user_input import ( validate_password, validate_publicname ) +from galaxy.structured_app import BasicApp, StructuredApp from galaxy.util.hash_util import new_secure_hash from galaxy.web import url_for @@ -69,7 +70,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): # TODO: incorp BaseAPIController.validate_in_users_and_groups # TODO: incorp CreatesApiKeysMixin # TODO: incorporate UsesFormDefinitionsMixin? - def __init__(self, app): + def __init__(self, app: BasicApp): self.model_class = app.model.User super().__init__(app) @@ -240,14 +241,6 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): except exceptions.ObjectNotFound: return None - def by_email_like(self, email_with_wildcards, filters=None, order_by=None, **kwargs): - """ - Find a user searching with SQL wildcards. - """ - filters = self._munge_filters(self.model_class.email.like(email_with_wildcards), filters) - order_by = order_by or (model.User.email, ) - return super().list(filters=filters, order_by=order_by, **kwargs) - def by_api_key(self, api_key, sa_session=None): """ Find a user by API key. @@ -353,6 +346,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): Create and return an API key for `user`. """ # TODO: seems like this should return the model + # Also TODO: seems unused? drop and see what happens? -John return api_keys.ApiKeyManager(self.app).create_api_key(user) def user_can_do_run_as(self, user) -> bool: @@ -617,7 +611,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): class UserSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): model_manager_class = UserManager - def __init__(self, app): + def __init__(self, app: StructuredApp): """ Convert a User and associated data to a dictionary representation. """ diff --git a/lib/galaxy/managers/visualizations.py b/lib/galaxy/managers/visualizations.py index 6564ab705d4..c7ae924061a 100644 --- a/lib/galaxy/managers/visualizations.py +++ b/lib/galaxy/managers/visualizations.py @@ -8,6 +8,7 @@ import logging from galaxy import model from galaxy.managers import sharable +from galaxy.structured_app import StructuredApp log = logging.getLogger(__name__) @@ -27,11 +28,6 @@ class VisualizationManager(sharable.SharableModelManager): annotation_assoc = model.VisualizationAnnotationAssociation rating_assoc = model.VisualizationRatingAssociation - def __init__(self, app, *args, **kwargs): - """ - """ - super().__init__(app, *args, **kwargs) - # def copy( self, trans, visualization, user, **kwargs ): # """ # """ @@ -45,7 +41,7 @@ class VisualizationSerializer(sharable.SharableModelSerializer): model_manager_class = VisualizationManager SINGLE_CHAR_ABBR = 'v' - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.visualization_manager = self.manager diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index e991b5458d5..aaf341d7edd 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -26,6 +26,7 @@ from galaxy import ( ) from galaxy.jobs.actions.post import ActionBox from galaxy.model.item_attrs import UsesAnnotations +from galaxy.structured_app import StructuredApp from galaxy.tools.parameters import ( params_to_incoming, visit_input_values @@ -68,7 +69,7 @@ class WorkflowsManager: the galaxy.workflow module. """ - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app def get_stored_workflow(self, trans, workflow_id, by_stored_id=True): @@ -305,7 +306,7 @@ CreatedWorkflow = namedtuple("CreatedWorkflow", ["stored_workflow", "workflow", class WorkflowContentsManager(UsesAnnotations): - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app self._resource_mapper_function = get_resource_mapper_function(app) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 35d4e990630..aef05f95c80 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -370,6 +370,7 @@ class JobLike: class User(Dictifiable, RepresentById): use_pbkdf2 = True bootstrap_admin_user = False + api_keys: 'List[APIKeys]' """ Data for a Galaxy user or admin and relations to their histories, credentials, and roles. diff --git a/lib/galaxy/model/base.py b/lib/galaxy/model/base.py index 019325e1984..cdf95d4a7f4 100644 --- a/lib/galaxy/model/base.py +++ b/lib/galaxy/model/base.py @@ -8,7 +8,7 @@ from inspect import ( getmembers, isclass ) -from typing import Dict +from typing import Dict, Type from sqlalchemy import event from sqlalchemy.orm import ( @@ -90,6 +90,18 @@ class ModelMapping(Bunch): return self.context +class SharedModelMapping(ModelMapping): + """Model mapping containing references to classes shared between Galaxy and ToolShed. + + Generally things can be more strongly typed when importing models directly, but we need + a way to do app.model. for common code shared by the tool shed and Galaxy. + """ + User: Type + GalaxySession: Type + APIKeys: Type + PasswordResetToken: Type + + def versioned_objects(iter): for obj in iter: if hasattr(obj, '__create_version__'): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index a0e0f904b66..e156fef42e4 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -5,7 +5,7 @@ are encapsulated here. import logging from threading import local -from typing import Optional +from typing import Optional, Type from sqlalchemy import ( and_, @@ -39,7 +39,7 @@ from sqlalchemy.sql import exists from sqlalchemy.types import BigInteger from galaxy import model -from galaxy.model.base import ModelMapping +from galaxy.model.base import SharedModelMapping from galaxy.model.custom_types import ( JSONType, MetadataType, @@ -2872,10 +2872,12 @@ def _workflow_invocation_update(self): model.WorkflowInvocation.update = _workflow_invocation_update # type: ignore -class GalaxyModelMapping(ModelMapping): +class GalaxyModelMapping(SharedModelMapping): security_agent: GalaxyRBACAgent thread_local_log: Optional[local] create_tables: bool + User: Type + GalaxySession: Type def init(file_path, url, engine_options=None, create_tables=False, map_install_models=False, diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 627c27c25a2..31c89cdec7d 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -1,6 +1,8 @@ import logging import re +from typing import Dict +from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.sql import select from sqlalchemy.sql.expression import func @@ -26,7 +28,7 @@ class TagHandler: Manages CRUD operations related to tagging objects. """ - def __init__(self, sa_session): + def __init__(self, sa_session: scoped_session) -> None: self.sa_session = sa_session # Minimum tag length. self.min_tag_len = 1 @@ -39,7 +41,7 @@ class TagHandler: # Key-value separator. self.key_value_separators = "=:" # Initialize with known classes - add to this in subclasses. - self.item_tag_assoc_info = {} + self.item_tag_assoc_info: Dict[str, ItemTagAssocInfo] = {} def create_tag_handler_session(self): # Creates a transient tag handler that avoids repeated flushes @@ -348,7 +350,7 @@ class TagHandler: class GalaxyTagHandler(TagHandler): - def __init__(self, sa_session): + def __init__(self, sa_session: scoped_session): from galaxy import model TagHandler.__init__(self, sa_session) self.item_tag_assoc_info["History"] = ItemTagAssocInfo(model.History, diff --git a/lib/galaxy/quota/__init__.py b/lib/galaxy/quota/__init__.py index e934dbf90db..d1ca9873593 100644 --- a/lib/galaxy/quota/__init__.py +++ b/lib/galaxy/quota/__init__.py @@ -1,5 +1,4 @@ """Galaxy Quotas""" -import abc import logging import galaxy.util @@ -7,7 +6,7 @@ import galaxy.util log = logging.getLogger(__name__) -class QuotaAgent(metaclass=abc.ABCMeta): +class QuotaAgent(): # metaclass=abc.ABCMeta """Abstraction around querying Galaxy for quota available and used. Certain parts of the app that deal directly with modifying the quota assume more than @@ -21,7 +20,7 @@ class QuotaAgent(metaclass=abc.ABCMeta): the quota in other apps (LDAP maybe?) or via configuration files. """ - @abc.abstractmethod + # TODO: make abstractmethod after they work better with mypy def get_quota(self, user): """Return quota in bytes or None if no quota is set.""" @@ -34,7 +33,7 @@ class QuotaAgent(metaclass=abc.ABCMeta): quota_str = 'unlimited' return quota_str - @abc.abstractmethod + # TODO: make abstractmethod after they work better with mypy def get_percent(self, trans=None, user=False, history=False, usage=False, quota=False): """Return the percentage of any storage quota applicable to the user/transaction.""" @@ -228,7 +227,8 @@ class DatabaseQuotaAgent(QuotaAgent): return False -def get_quota_agent(config, model): +def get_quota_agent(config, model) -> QuotaAgent: + quota_agent: QuotaAgent if config.enable_quotas: quota_agent = galaxy.quota.DatabaseQuotaAgent(model) else: diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index 81f337a322c..7728929a0ee 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -4,13 +4,16 @@ from typing import Any, Optional from kombu import Connection from galaxy.auth import AuthManager +from galaxy.datatypes.registry import Registry +from galaxy.di import Container from galaxy.files import ConfiguredFileSources from galaxy.job_metrics import JobMetrics -from galaxy.model.base import ModelMapping +from galaxy.model.base import ModelMapping, SharedModelMapping from galaxy.model.mapping import GalaxyModelMapping from galaxy.model.security import GalaxyRBACAgent from galaxy.model.security import HostAgent from galaxy.model.tags import GalaxyTagHandler +from galaxy.quota import QuotaAgent from galaxy.security.idencoding import IdEncodingHelper from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify import test_data @@ -20,7 +23,25 @@ from galaxy.webhooks import WebhooksRegistry from galaxy.workflow.trs_proxy import TrsProxy -class StructuredApp: +class BasicApp(Container): + """Stripped down version of the ``app`` shared between Galaxy and ToolShed. + + Code that is shared between Galaxy and the Tool Shed should be annotated as + using BasicApp instead of StructuredApp below. + """ + name: str + config: Any + application_stack: ApplicationStack + model: SharedModelMapping + security: IdEncodingHelper + auth_manager: AuthManager + toolbox: Any + security_agent: Any + quota_agent: QuotaAgent + datatypes_registry: Registry + + +class StructuredApp(BasicApp): """Interface defining typed description of the Galaxy UniverseApplication. Ideally nothing that depends on StructuredApp should require @@ -31,12 +52,9 @@ class StructuredApp: (cyclical imports), we're just setting the class attributes to Any. """ - name: str is_webapp: bool # is_webapp will be set to true when building WSGI app new_installation: bool - config: Any tag_handler: GalaxyTagHandler - application_stack: ApplicationStack amqp_internal_connection_obj: Optional[Connection] dependency_resolvers_view: DependencyResolversView test_data_resolver: test_data.TestDataResolver @@ -45,10 +63,8 @@ class StructuredApp: job_metrics: JobMetrics model: GalaxyModelMapping install_model: ModelMapping - security: IdEncodingHelper security_agent: GalaxyRBACAgent host_security_agent: HostAgent - auth_manager: AuthManager trs_proxy: TrsProxy webhooks_registry: WebhooksRegistry @@ -73,6 +89,10 @@ class StructuredApp: installed_repository_manager: Any # 'galaxy.tool_shed.galaxy_install.installed_repository_manager.InstalledRepositoryManager' workflow_scheduling_manager: Any # 'galaxy.workflow.scheduling_manager.WorkflowSchedulingManager' interactivetool_manager: Any + job_manager: Any # galaxy.jobs.manager.JobManager user_manager: Any api_keys_manager: Any - toolbox: Any + + @property + def is_job_handler(self) -> bool: + pass diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index f8fd322ab8a..26a9e2f46d7 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -27,7 +27,6 @@ from galaxy import ( ) from galaxy.exceptions import ToolInputsNotReadyException from galaxy.job_execution import output_collect -from galaxy.managers.jobs import JobSearch from galaxy.metadata import get_metadata_compute_strategy from galaxy.tool_shed.util.repository_util import get_installed_repository from galaxy.tool_shed.util.shed_util_common import set_image_paths @@ -553,7 +552,7 @@ class Tool(Dictifiable): # The job search is only relevant in a galaxy context, and breaks # loading tools into the toolshed for validation. if self.app.name == 'galaxy': - self.job_search = JobSearch(app=self.app) + self.job_search = self.app.job_search def __getattr__(self, name): lazy_attributes = { diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index 501c8d4641f..94e0b6d2996 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -7,6 +7,7 @@ import tempfile import zlib from collections import defaultdict from threading import Lock +from typing import Dict, List, Tuple from sqlalchemy.orm import ( defer, @@ -14,9 +15,13 @@ from sqlalchemy.orm import ( ) from sqlitedict import SqliteDict +from galaxy.model.tool_shed_install import ToolShedRepository +from galaxy.structured_app import StructuredApp +from galaxy.tools.toolbox.base import ToolConfRepository from galaxy.util import unicodify from galaxy.util.hash_util import md5_hash_file + log = logging.getLogger(__name__) CURRENT_TOOL_CACHE_VERSION = 0 @@ -277,8 +282,11 @@ class ToolShedRepositoryCache: """ Cache installed ToolShedRepository objects. """ + local_repositories: List[ToolConfRepository] + repositories: List[ToolShedRepository] + repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]] - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app # Contains ToolConfRepository objects created from shed_tool_conf.xml entries self.local_repositories = [] @@ -294,10 +302,10 @@ class ToolShedRepositoryCache: def rebuild(self): try: session = self.app.install_model._SessionLocal() - self.repositories = session.query(self.app.install_model.ToolShedRepository).options( - defer(self.app.install_model.ToolShedRepository.metadata), + self.repositories = session.query(ToolShedRepository).options( + defer(ToolShedRepository.metadata), joinedload('tool_dependencies').subqueryload('tool_shed_repository').options( - defer(self.app.install_model.ToolShedRepository.metadata) + defer(ToolShedRepository.metadata) ), ).all() repos_by_tuple = defaultdict(list) diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index a8891249288..4805f1320ab 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -2,9 +2,10 @@ import errno import json import logging import os - +from typing import Dict from galaxy import util +from galaxy.structured_app import StructuredApp from galaxy.tools.data import TabularToolDataTable from galaxy.util.template import fill_template @@ -16,7 +17,10 @@ DEFAULT_VALUE_TRANSLATION_TYPE = 'template' class DataManagers: - def __init__(self, app, xml_filename=None): + data_managers: Dict[str, 'DataManager'] + managed_data_tables: Dict[str, 'DataManager'] + + def __init__(self, app: StructuredApp, xml_filename=None): self.app = app self.data_managers = {} self.managed_data_tables = {} diff --git a/lib/galaxy/tools/repositories.py b/lib/galaxy/tools/repositories.py index a843a18e725..ed0ac21601e 100644 --- a/lib/galaxy/tools/repositories.py +++ b/lib/galaxy/tools/repositories.py @@ -40,6 +40,7 @@ class ValidationContext: # not appear to be an issue with the len_file_path config option. self.config.builds_file_path = None self.genome_builds = GenomeBuilds(self) + self.job_search = None def __enter__(self): return self diff --git a/lib/galaxy/visualization/genomes.py b/lib/galaxy/visualization/genomes.py index 8c8e265cb39..1737b9e1802 100644 --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -3,6 +3,7 @@ import os import re import sys from json import loads +from typing import Dict from bx.seq.twobit import TwoBitFile @@ -10,6 +11,7 @@ from galaxy.exceptions import ( ObjectNotFound, ReferenceDataError, ) +from galaxy.structured_app import StructuredApp from galaxy.util.bunch import Bunch log = logging.getLogger(__name__) @@ -195,10 +197,10 @@ class Genomes: Provides information about available genome data and methods for manipulating that data. """ - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app # Create list of genomes from app.genome_builds - self.genomes = {} + self.genomes: Dict[str, Genome] = {} # Store internal versions of data tables for twobit and __dbkey__ self._table_versions = {'twobit': None, '__dbkeys__': None} self.reload_genomes() diff --git a/lib/galaxy/visualization/plugins/resource_parser.py b/lib/galaxy/visualization/plugins/resource_parser.py index 58d5893db35..bdc0cca9d00 100644 --- a/lib/galaxy/visualization/plugins/resource_parser.py +++ b/lib/galaxy/visualization/plugins/resource_parser.py @@ -44,8 +44,8 @@ class ResourceParser: def _init_managers(self, app): return bunch.Bunch( - visualization=visualization_manager.VisualizationManager(app), - hda=hda_manager.HDAManager(app) + visualization=app[visualization_manager.VisualizationManager], + hda=app[hda_manager.HDAManager], ) def parse_parameter_dictionary(self, trans, param_config_dict, query_params, param_modifiers=None): diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index ea8c8f51fcf..0f25cabb406 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -2,7 +2,7 @@ Contains functionality needed in every web interface """ import logging -from typing import Optional +from typing import Any, Optional from sqlalchemy import true from webob.exc import ( @@ -34,7 +34,6 @@ from galaxy.model import ( tags, ) from galaxy.model.item_attrs import UsesAnnotations -from galaxy.structured_app import StructuredApp from galaxy.util.dictifiable import Dictifiable from galaxy.util.sanitize_html import sanitize_html from galaxy.web import ( @@ -58,9 +57,8 @@ class BaseController: """ Base class for Galaxy web application controllers. """ - app: StructuredApp - def __init__(self, app: StructuredApp) -> None: + def __init__(self, app): """Initialize an interface for application 'app'""" self.app = app self.sa_session = app.model.context @@ -1346,8 +1344,8 @@ class UsesFormDefinitionsMixin: class SharableMixin: """ Mixin for a controller that manages an item that can be shared. """ - manager = None - serializer = None + manager: Any = None + serializer: Any = None # -- Implemented methods. -- diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 50e9c07ac6f..5e59db65f74 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -77,6 +77,7 @@ class WebApplication(base.WebApplication): * builds mako template lookups. * generates GalaxyWebTransactions. """ + injection_aware: bool = False def __init__(self, galaxy_app, session_cookie='galaxysession', name=None): self.name = name @@ -152,7 +153,17 @@ class WebApplication(base.WebApplication): """ Extension point, allow apps to construct controllers differently, really just used to stub out actual controllers for routes testing. """ - return T(app) + controller = None + if self.injection_aware: + controller = app.resolve_or_none(T) + if controller is not None: + for key, value in T.__dict__.items(): + if hasattr(value, "galaxy_type_depends"): + value_type = value.galaxy_type_depends + setattr(controller, key, app[value_type]) + if controller is None: + controller = T(app) + return controller def config_allows_origin(origin_raw, config): @@ -187,8 +198,8 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, context.ProvidesHistoryCo def __init__(self, environ: Dict[str, Any], app, webapp, session_cookie=None) -> None: self._app = app self.webapp = webapp - self.user_manager = UserManager(app) - self.session_manager = GalaxySessionManager(app.model) + self.user_manager = app[UserManager] + self.session_manager = app[GalaxySessionManager] base.DefaultWebTransaction.__init__(self, environ) self.setup_i18n() self.expunge_all() diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index e96ee88169a..78b246dd400 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -2,45 +2,52 @@ This module *does not* contain API routes. It exclusively contains dependencies to be used in FastAPI routes """ from typing import ( + Any, AsyncGenerator, cast, Optional, + Type, + TypeVar, ) from fastapi import ( Cookie, - Depends, Header, Query, ) -from sqlalchemy.orm import Session +from fastapi.params import Depends try: from starlette_context import context as request_context except ImportError: request_context = None +from starlette.requests import Request from galaxy import ( app as galaxy_app, model, ) -from galaxy.app import UniverseApplication from galaxy.exceptions import ( AdminRequiredException, UserCannotRunAsException, UserInvalidRunAsException, ) -from galaxy.managers.jobs import JobManager from galaxy.managers.session import GalaxySessionManager from galaxy.managers.users import UserManager from galaxy.model import User from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.security.idencoding import IdEncodingHelper +from galaxy.structured_app import StructuredApp from galaxy.web.framework.decorators import require_admin_message +from galaxy.webapps.base.controller import BaseAPIController from galaxy.work.context import SessionRequestContext -async def _get_app() -> AsyncGenerator[UniverseApplication, None]: - app = cast(UniverseApplication, galaxy_app.app) +def get_app() -> StructuredApp: + return cast(StructuredApp, galaxy_app.app) + + +async def get_app_with_request_session() -> AsyncGenerator[StructuredApp, None]: + app = get_app() request_id = request_context.data['X-Request-ID'] app.model.set_request_id(request_id) try: @@ -49,34 +56,35 @@ async def _get_app() -> AsyncGenerator[UniverseApplication, None]: app.model.unset_request_id(request_id) -async def get_app(app=Depends(_get_app)) -> UniverseApplication: - return app +DependsOnApp = Depends(get_app_with_request_session) -def get_id_encoding_helper(app: UniverseApplication = Depends(get_app)) -> IdEncodingHelper: - return app.security +T = TypeVar("T") -def get_job_manager(app: UniverseApplication = Depends(get_app)) -> JobManager: - return JobManager(app=app) +class GalaxyTypeDepends(Depends): + """Variant of fastapi Depends that can also work on WSGI Galaxy controllers.""" + + def __init__(self, callable, dep_type): + super().__init__(callable) + self.galaxy_type_depends = dep_type -def get_db(app: UniverseApplication = Depends(get_app)) -> Session: - # TODO: return sqlachemy 2.0 style session without autocommit and expire_on_commit! - return app.model.session +def depends(dep_type: Type[T]) -> Any: + + def _do_resolve(request: Request): + return get_app().resolve(dep_type) + + return GalaxyTypeDepends(_do_resolve, dep_type) -def get_user_manager(app: UniverseApplication = Depends(get_app)) -> UserManager: - return UserManager(app) - - -def get_session_manager(app: UniverseApplication = Depends(get_app)) -> GalaxySessionManager: +def get_session_manager(app: StructuredApp = DependsOnApp) -> GalaxySessionManager: # TODO: find out how to adapt dependency for Galaxy/Report/TS return GalaxySessionManager(app.model) def get_session(session_manager: GalaxySessionManager = Depends(get_session_manager), - security: IdEncodingHelper = Depends(get_id_encoding_helper), + security: IdEncodingHelper = depends(IdEncodingHelper), galaxysession: Optional[str] = Cookie(None)) -> Optional[model.GalaxySession]: if galaxysession: session_key = security.decode_guid(galaxysession) @@ -87,8 +95,8 @@ def get_session(session_manager: GalaxySessionManager = Depends(get_session_mana def get_api_user( - security: IdEncodingHelper = Depends(get_id_encoding_helper), - user_manager: UserManager = Depends(get_user_manager), + security: IdEncodingHelper = depends(IdEncodingHelper), + user_manager: UserManager = depends(UserManager), key: Optional[str] = Query(None), x_api_key: Optional[str] = Header(None), run_as: Optional[EncodedDatabaseIdField] = Header(None, title='Run as User', description='Admins and ')) -> Optional[User]: @@ -114,13 +122,28 @@ def get_user(galaxy_session: Optional[model.GalaxySession] = Depends(get_session return api_user -def get_trans(app: UniverseApplication = Depends(get_app), user: Optional[User] = Depends(get_user), +DependsOnUser = Depends(get_user) + + +def get_trans(app: StructuredApp = DependsOnApp, user: Optional[User] = Depends(get_user), galaxy_session: Optional[model.GalaxySession] = Depends(get_session), ) -> SessionRequestContext: return SessionRequestContext(app=app, user=user, galaxy_session=galaxy_session) -def get_admin_user(trans: SessionRequestContext = Depends(get_trans)): +DependsOnTrans = Depends(get_trans) + + +def get_admin_user(trans: SessionRequestContext = DependsOnTrans): if not trans.user_is_admin: raise AdminRequiredException(require_admin_message(trans.app.config, trans.user)) return trans.user + + +AdminUserRequired = Depends(get_admin_user) + + +class BaseGalaxyAPIController(BaseAPIController): + + def __init__(self, app: StructuredApp): + super().__init__(app) diff --git a/lib/galaxy/webapps/galaxy/api/annotations.py b/lib/galaxy/webapps/galaxy/api/annotations.py index b9d323c90d4..5f93f54db51 100644 --- a/lib/galaxy/webapps/galaxy/api/annotations.py +++ b/lib/galaxy/webapps/galaxy/api/annotations.py @@ -13,14 +13,14 @@ from galaxy.model.item_attrs import UsesAnnotations from galaxy.util.sanitize_html import sanitize_html from galaxy.web import expose_api from galaxy.webapps.base.controller import ( - BaseAPIController, UsesStoredWorkflowMixin ) +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class BaseAnnotationsController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnnotations): +class BaseAnnotationsController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, UsesAnnotations): tagged_item_id: str @expose_api @@ -65,10 +65,7 @@ class BaseAnnotationsController(BaseAPIController, UsesStoredWorkflowMixin, Uses class HistoryAnnotationsController(BaseAnnotationsController): controller_name = "history_annotations" tagged_item_id = "history_id" - - def __init__(self, app): - super().__init__(app) - self.history_manager = managers.histories.HistoryManager(app) + history_manager: managers.histories.HistoryManager = depends(managers.histories.HistoryManager) def _get_item_from_id(self, trans: ProvidesHistoryContext, idstr): decoded_idstr = self.decode_id(idstr) @@ -79,10 +76,7 @@ class HistoryAnnotationsController(BaseAnnotationsController): class HistoryContentAnnotationsController(BaseAnnotationsController): controller_name = "history_content_annotations" tagged_item_id = "history_content_id" - - def __init__(self, app): - super().__init__(app) - self.hda_manager = managers.hdas.HDAManager(app) + hda_manager: managers.hdas.HDAManager = depends(managers.hdas.HDAManager) def _get_item_from_id(self, trans: ProvidesHistoryContext, idstr): decoded_idstr = self.decode_id(idstr) diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index 52a08de6239..b8b96fe0dc5 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -25,13 +25,13 @@ from galaxy.util import ( unicodify ) from galaxy.web import expose_api_anonymous_and_sessionless -from galaxy.webapps.base.controller import BaseAPIController from galaxy.webapps.base.webapp import GalaxyWebTransaction +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class AuthenticationController(BaseAPIController): +class AuthenticationController(BaseGalaxyAPIController): @expose_api_anonymous_and_sessionless def options(self, trans: GalaxyWebTransaction, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 2c9034d2d3f..fd81b6654b2 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -10,21 +10,22 @@ from galaxy.managers import ( cloud, datasets ) +from galaxy.structured_app import StructuredApp from galaxy.web import expose_api -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class CloudController(BaseAPIController): +class CloudController(BaseGalaxyAPIController): """ RESTfull controller for interaction with Amazon S3. """ - def __init__(self, app): + def __init__(self, app: StructuredApp, cloud_manager: cloud.CloudManager, datasets_serializer: datasets.DatasetSerializer): super().__init__(app) - self.cloud_manager = cloud.CloudManager(app) - self.datasets_serializer = datasets.DatasetSerializer(app) + self.cloud_manager = cloud_manager + self.datasets_serializer = datasets_serializer @expose_api def index(self, trans, **kwargs): diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 447d14752aa..95e7e723a56 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -18,21 +18,22 @@ from galaxy.exceptions import ( RequestParameterMissingException ) from galaxy.managers import cloudauthzs +from galaxy.structured_app import StructuredApp from galaxy.util import unicodify from galaxy.web import ( expose_api ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class CloudAuthzController(BaseAPIController): +class CloudAuthzController(BaseGalaxyAPIController): """ RESTfull controller for defining cloud authorizations. """ - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.cloudauthz_manager = cloudauthzs.CloudAuthzManager(app) self.cloudauthz_serializer = cloudauthzs.CloudAuthzsSerializer(app) diff --git a/lib/galaxy/webapps/galaxy/api/configuration.py b/lib/galaxy/webapps/galaxy/api/configuration.py index ac9317a869e..fcbaf897882 100644 --- a/lib/galaxy/webapps/galaxy/api/configuration.py +++ b/lib/galaxy/webapps/galaxy/api/configuration.py @@ -10,16 +10,12 @@ from typing import ( Optional ) -from fastapi import ( - Depends, - Path, -) +from fastapi import Path from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter from galaxy.managers.configuration import ConfigurationManager from galaxy.managers.context import ( - ProvidesAppContext, ProvidesUserContext, ) from galaxy.managers.users import ( @@ -28,18 +24,17 @@ from galaxy.managers.users import ( ) from galaxy.model import User from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import StructuredApp from galaxy.web import ( expose_api, expose_api_anonymous_and_sessionless, require_admin ) -from galaxy.webapps.base.controller import BaseAPIController from . import ( - get_admin_user, - get_app, - get_trans, - get_user, + AdminUserRequired, + BaseGalaxyAPIController, + depends, + DependsOnTrans, + DependsOnUser, ) from .common import ( parse_serialization_params, @@ -51,7 +46,6 @@ log = logging.getLogger(__name__) router = APIRouter(tags=['configuration']) -AdminUserRequired = Depends(get_admin_user) EncodedIdPathParam: EncodedDatabaseIdField = Path( ..., @@ -60,20 +54,16 @@ EncodedIdPathParam: EncodedDatabaseIdField = Path( ) -def get_configuration_manager(app: StructuredApp = Depends(get_app)) -> ConfigurationManager: - return ConfigurationManager(app) - - @cbv(router) class FastAPIConfiguration: - configuration_manager: ConfigurationManager = Depends(get_configuration_manager) + configuration_manager: ConfigurationManager = depends(ConfigurationManager) @router.get( '/api/whoami', summary="Return information about the current authenticated user", response_description="Information about the current authenticated user" ) - def whoami(self, user: User = Depends(get_user)) -> Optional[UserModel]: + def whoami(self, user: User = DependsOnUser) -> Optional[UserModel]: """Return information about the current authenticated user.""" return _user_to_model(user) @@ -84,7 +74,7 @@ class FastAPIConfiguration: ) def index( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, view: Optional[str] = SerializationViewQueryParam, keys: Optional[str] = SerializationKeysQueryParam, ) -> Dict[str, Any]: @@ -124,7 +114,6 @@ class FastAPIConfiguration: ) def decode_id( self, - trans: ProvidesAppContext = Depends(get_trans), encoded_id: EncodedDatabaseIdField = EncodedIdPathParam ) -> Dict[str, int]: """Decode a given id.""" @@ -150,14 +139,12 @@ class FastAPIConfiguration: self.configuration_manager.reload_toolbox() -class ConfigurationController(BaseAPIController): - - def __init__(self, app: StructuredApp): - super().__init__(app) - self.configuration_manager = ConfigurationManager(app) +class ConfigurationController(BaseGalaxyAPIController): + configuration_manager: ConfigurationManager = depends(ConfigurationManager) + user_manager: UserManager = depends(UserManager) @expose_api - def whoami(self, trans, **kwd): + def whoami(self, trans: ProvidesUserContext, **kwd): """ GET /api/whoami Return information about the current authenticated user. @@ -165,7 +152,7 @@ class ConfigurationController(BaseAPIController): :returns: dictionary with user information :rtype: dict """ - user = UserManager(self.app).current_user(trans) + user = self.user_manager.current_user(trans) return _user_to_model(user) @expose_api_anonymous_and_sessionless diff --git a/lib/galaxy/webapps/galaxy/api/container_resolution.py b/lib/galaxy/webapps/galaxy/api/container_resolution.py index d545c40cecf..208c1195505 100644 --- a/lib/galaxy/webapps/galaxy/api/container_resolution.py +++ b/lib/galaxy/webapps/galaxy/api/container_resolution.py @@ -5,19 +5,20 @@ import logging import requests +from galaxy.structured_app import StructuredApp from galaxy.tool_util.deps import views from galaxy.web import ( expose_api, require_admin ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class ContainerResolutionAPIController(BaseAPIController): +class ContainerResolutionAPIController(BaseGalaxyAPIController): - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self._view = views.ContainerResolutionView(app) diff --git a/lib/galaxy/webapps/galaxy/api/dataset_collections.py b/lib/galaxy/webapps/galaxy/api/dataset_collections.py index 4de24c853e7..9a998f49063 100644 --- a/lib/galaxy/webapps/galaxy/api/dataset_collections.py +++ b/lib/galaxy/webapps/galaxy/api/dataset_collections.py @@ -11,23 +11,19 @@ from galaxy.managers.collections_util import ( dictify_element_reference ) from galaxy.managers.context import ProvidesHistoryContext +from galaxy.managers.histories import HistoryManager from galaxy.web import expose_api -from galaxy.webapps.base.controller import ( - BaseAPIController, - UsesLibraryMixinItems -) +from galaxy.webapps.base.controller import UsesLibraryMixinItems +from . import BaseGalaxyAPIController, depends log = getLogger(__name__) class DatasetCollectionsController( - BaseAPIController, + BaseGalaxyAPIController, UsesLibraryMixinItems, ): - - def __init__(self, app): - super().__init__(app) - self.history_manager = app.history_manager + history_manager: HistoryManager = depends(HistoryManager) @expose_api def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 74b1d3c64df..dd0040b5eae 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -7,12 +7,17 @@ import os from galaxy import ( exceptions as galaxy_exceptions, - managers, model, util, web ) from galaxy.datatypes import dataproviders +from galaxy.managers.hdas import HDAManager, HDASerializer +from galaxy.managers.hdcas import HDCASerializer +from galaxy.managers.histories import HistoryManager +from galaxy.managers.history_contents import HistoryContentsFilters +from galaxy.managers.history_contents import HistoryContentsManager +from galaxy.managers.lddas import LDDAManager from galaxy.util.path import ( safe_walk ) @@ -22,26 +27,24 @@ from galaxy.visualization.data_providers.genome import ( SamDataProvider ) from galaxy.web.framework.helpers import is_true -from galaxy.webapps.base.controller import ( - BaseAPIController, - UsesVisualizationMixin -) +from galaxy.webapps.base.controller import UsesVisualizationMixin +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class DatasetsController(BaseAPIController, UsesVisualizationMixin): +class DatasetsController(BaseGalaxyAPIController, UsesVisualizationMixin): + history_manager: HistoryManager = depends(HistoryManager) + hda_manager: HDAManager = depends(HDAManager) + hda_serializer: HDASerializer = depends(HDASerializer) + hdca_serializer: HDCASerializer = depends(HDCASerializer) + ldda_manager: LDDAManager = depends(LDDAManager) + history_contents_manager: HistoryContentsManager = depends(HistoryContentsManager) + history_contents_filters: HistoryContentsFilters = depends(HistoryContentsFilters) - def __init__(self, app): - super().__init__(app) - self.history_manager = managers.histories.HistoryManager(app) - self.hda_manager = managers.hdas.HDAManager(app) - self.hda_serializer = managers.hdas.HDASerializer(app) - self.hdca_serializer = managers.hdcas.HDCASerializer(app) - self.serializer_by_type = {'dataset': self.hda_serializer, 'dataset_collection': self.hdca_serializer} - self.ldda_manager = managers.lddas.LDDAManager(app) - self.history_contents_manager = managers.history_contents.HistoryContentsManager(app) - self.history_contents_filters = managers.history_contents.HistoryContentsFilters(app) + @property + def serializer_by_type(self): + return {'dataset': self.hda_serializer, 'dataset_collection': self.hdca_serializer} def _parse_serialization_params(self, kwd, default_view): view = kwd.get('view', None) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index bd089330d5d..fc262e9ac7f 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -9,14 +9,10 @@ from typing import ( Union, ) -from fastapi import ( - Depends, - Query, -) +from fastapi import Query from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter -from galaxy.app import UniverseApplication from galaxy.datatypes.registry import Registry from galaxy.managers.datatypes import ( DatatypeConverterList, @@ -33,8 +29,10 @@ from galaxy.managers.datatypes import ( ) from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless -from galaxy.webapps.base.controller import BaseAPIController -from . import get_app +from . import ( + BaseGalaxyAPIController, + depends, +) log = logging.getLogger(__name__) @@ -53,13 +51,9 @@ UploadOnlyQueryParam: Optional[bool] = Query( ) -def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Registry: - return app.datatypes_registry - - @cbv(router) class FastAPIDatatypes: - datatypes_registry: Registry = Depends(get_datatypes_registry) + datatypes_registry: Registry = depends(Registry) @router.get( '/api/datatypes', @@ -138,7 +132,7 @@ class FastAPIDatatypes: return self.datatypes_registry.edam_data -class DatatypesController(BaseAPIController): +class DatatypesController(BaseGalaxyAPIController): @expose_api_anonymous_and_sessionless def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/display_applications.py b/lib/galaxy/webapps/galaxy/api/display_applications.py index 56adeaf3ae3..94cb0957d9b 100644 --- a/lib/galaxy/webapps/galaxy/api/display_applications.py +++ b/lib/galaxy/webapps/galaxy/api/display_applications.py @@ -3,22 +3,18 @@ API operations on annotations. """ import logging -from galaxy.app import StructuredApp from galaxy.managers.display_applications import DisplayApplicationsManager from galaxy.web import ( expose_api, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class DisplayApplicationsController(BaseAPIController): - - def __init__(self, app: StructuredApp): - super().__init__(app) - self.manager = DisplayApplicationsManager(app) +class DisplayApplicationsController(BaseGalaxyAPIController): + manager = depends(DisplayApplicationsManager) @expose_api def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py index 46f6f0f1996..037a0e47fe5 100644 --- a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py +++ b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py @@ -17,9 +17,6 @@ class DynamicToolsController(BaseAPIController): to run these tools and view functional information. """ - def __init__(self, app): - super().__init__(app) - @expose_api_anonymous_and_sessionless def index(self, trans, **kwds): """ diff --git a/lib/galaxy/webapps/galaxy/api/extended_metadata.py b/lib/galaxy/webapps/galaxy/api/extended_metadata.py index 0c561cf7404..f8068d9fd41 100644 --- a/lib/galaxy/webapps/galaxy/api/extended_metadata.py +++ b/lib/galaxy/webapps/galaxy/api/extended_metadata.py @@ -8,17 +8,17 @@ from galaxy import ( web ) from galaxy.webapps.base.controller import ( - BaseAPIController, HTTPNotImplemented, UsesExtendedMetadataMixin, UsesLibraryMixinItems, UsesStoredWorkflowMixin ) +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class BaseExtendedMetadataController(BaseAPIController, UsesExtendedMetadataMixin, UsesLibraryMixinItems, UsesStoredWorkflowMixin): +class BaseExtendedMetadataController(BaseGalaxyAPIController, UsesExtendedMetadataMixin, UsesLibraryMixinItems, UsesStoredWorkflowMixin): @web.legacy_expose_api def index(self, trans, **kwd): @@ -75,10 +75,7 @@ class LibraryDatasetExtendMetadataController(BaseExtendedMetadataController): class HistoryDatasetExtendMetadataController(BaseExtendedMetadataController): controller_name = "history_dataset_extended_metadata" exmeta_item_id = "history_content_id" - - def __init__(self, app): - super().__init__(app) - self.hda_manager = managers.hdas.HDAManager(app) + hda_manager: managers.hdas.HDAManager = depends(managers.hdas.HDAManager) def _get_item_from_id(self, trans, idstr, check_writable=True): decoded_idstr = self.decode_id(idstr) diff --git a/lib/galaxy/webapps/galaxy/api/folder_contents.py b/lib/galaxy/webapps/galaxy/api/folder_contents.py index 984fc0d61cb..0a9b0d7666b 100644 --- a/lib/galaxy/webapps/galaxy/api/folder_contents.py +++ b/lib/galaxy/webapps/galaxy/api/folder_contents.py @@ -5,29 +5,27 @@ import logging from galaxy import ( exceptions, - managers, util ) -from galaxy.managers import folders +from galaxy.managers.folders import FolderManager +from galaxy.managers.hdas import HDAManager from galaxy.model import tags from galaxy.web import ( expose_api, expose_api_anonymous ) -from galaxy.webapps.base.controller import BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems +from galaxy.webapps.base.controller import UsesLibraryMixin, UsesLibraryMixinItems +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems): +class FolderContentsController(BaseGalaxyAPIController, UsesLibraryMixin, UsesLibraryMixinItems): """ Class controls retrieval, creation and updating of folder contents. """ - - def __init__(self, app): - super().__init__(app) - self.folder_manager = folders.FolderManager() - self.hda_manager = managers.hdas.HDAManager(app) + hda_manager: HDAManager = depends(HDAManager) + folder_manager: FolderManager = depends(FolderManager) @expose_api_anonymous def index(self, trans, folder_id, limit=None, offset=None, search_text=None, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/folders.py b/lib/galaxy/webapps/galaxy/api/folders.py index a7c01e02e30..2c13f77e206 100644 --- a/lib/galaxy/webapps/galaxy/api/folders.py +++ b/lib/galaxy/webapps/galaxy/api/folders.py @@ -8,15 +8,17 @@ from galaxy import ( util ) from galaxy.managers import folders, roles +from galaxy.structured_app import StructuredApp from galaxy.web import expose_api -from galaxy.webapps.base.controller import BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems +from galaxy.webapps.base.controller import UsesLibraryMixin, UsesLibraryMixinItems +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class FoldersController(BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems): +class FoldersController(BaseGalaxyAPIController, UsesLibraryMixin, UsesLibraryMixinItems): - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.folder_manager = folders.FolderManager() self.role_manager = roles.RoleManager(app) diff --git a/lib/galaxy/webapps/galaxy/api/forms.py b/lib/galaxy/webapps/galaxy/api/forms.py index 2a112c2e9ce..1ca8588d08f 100644 --- a/lib/galaxy/webapps/galaxy/api/forms.py +++ b/lib/galaxy/webapps/galaxy/api/forms.py @@ -6,12 +6,12 @@ import logging from galaxy import web from galaxy.forms.forms import form_factory from galaxy.util import XML -from galaxy.webapps.base.controller import BaseAPIController, url_for - +from galaxy.webapps.base.controller import url_for +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class FormDefinitionAPIController(BaseAPIController): +class FormDefinitionAPIController(BaseGalaxyAPIController): @web.legacy_expose_api def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/genomes.py b/lib/galaxy/webapps/galaxy/api/genomes.py index 28f738824e4..780bcf4c4ae 100644 --- a/lib/galaxy/webapps/galaxy/api/genomes.py +++ b/lib/galaxy/webapps/galaxy/api/genomes.py @@ -4,7 +4,6 @@ from typing import ( ) from fastapi import ( - Depends, Path, Query, ) @@ -14,17 +13,13 @@ from fastapi_utils.inferring_router import InferringRouter as APIRouter from galaxy.managers.context import ProvidesUserContext from galaxy.managers.genomes import GenomesManager -from galaxy.structured_app import StructuredApp from galaxy.web import ( expose_api_anonymous, expose_api_raw_anonymous, ) from galaxy.web.framework.helpers import is_true -from galaxy.webapps.base.controller import BaseAPIController -from . import ( - get_app, - get_trans, -) +from . import BaseGalaxyAPIController, depends, DependsOnTrans + router = APIRouter(tags=['genomes']) @@ -83,10 +78,6 @@ IndexTypeQueryParam: str = Query( ) -def get_genomes_manager(app: StructuredApp = Depends(get_app)) -> GenomesManager: - return GenomesManager(app) - - def get_id(base, format): if format: return f"{base}.{format}" @@ -95,7 +86,7 @@ def get_id(base, format): @cbv(router) class FastAPIGenomes: - manager: GenomesManager = Depends(get_genomes_manager) + manager: GenomesManager = depends(GenomesManager) @router.get( '/api/genomes', @@ -104,7 +95,7 @@ class FastAPIGenomes: ) def index( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, chrom_info: bool = ChromInfoQueryParam ) -> List[List[str]]: return self.manager.get_dbkeys(trans.user, chrom_info) @@ -116,7 +107,7 @@ class FastAPIGenomes: ) def show( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, id: str = IdPathParam, reference: bool = ReferenceQueryParam, num: int = NumQueryParam, @@ -150,7 +141,7 @@ class FastAPIGenomes: ) def sequences( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, id: str = IdPathParam, reference: bool = ReferenceQueryParam, chrom: str = ChromQueryParam, @@ -163,13 +154,11 @@ class FastAPIGenomes: return Response(rval) -class GenomesController(BaseAPIController): +class GenomesController(BaseGalaxyAPIController): """ RESTful controller for interactions with genome data. """ - def __init__(self, app: StructuredApp): - super().__init__(app) - self.manager = GenomesManager(app) + manager: GenomesManager = depends(GenomesManager) @expose_api_anonymous def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py index 3be7ed95880..2e2410bf3ae 100644 --- a/lib/galaxy/webapps/galaxy/api/group_roles.py +++ b/lib/galaxy/webapps/galaxy/api/group_roles.py @@ -10,16 +10,13 @@ from galaxy.web import ( expose_api, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class GroupRolesAPIController(BaseAPIController): - - def __init__(self, app): - super().__init__(app) - self.manager = GroupRolesManager(app) +class GroupRolesAPIController(BaseGalaxyAPIController): + manager = depends(GroupRolesManager) @require_admin @expose_api diff --git a/lib/galaxy/webapps/galaxy/api/group_users.py b/lib/galaxy/webapps/galaxy/api/group_users.py index b1fb16b3235..ec75947701d 100644 --- a/lib/galaxy/webapps/galaxy/api/group_users.py +++ b/lib/galaxy/webapps/galaxy/api/group_users.py @@ -10,16 +10,13 @@ from galaxy.web import ( expose_api, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class GroupUsersAPIController(BaseAPIController): - - def __init__(self, app): - super().__init__(app) - self.manager = GroupUsersManager(app) +class GroupUsersAPIController(BaseGalaxyAPIController): + manager = depends(GroupUsersManager) @require_admin @expose_api diff --git a/lib/galaxy/webapps/galaxy/api/groups.py b/lib/galaxy/webapps/galaxy/api/groups.py index 9433dab6ff6..5e18a0f8c44 100644 --- a/lib/galaxy/webapps/galaxy/api/groups.py +++ b/lib/galaxy/webapps/galaxy/api/groups.py @@ -7,7 +7,6 @@ from typing import ( Dict, ) -from galaxy.app import StructuredApp from galaxy.managers.context import ProvidesAppContext from galaxy.managers.groups import GroupsManager from galaxy.schema.fields import EncodedDatabaseIdField @@ -15,16 +14,13 @@ from galaxy.web import ( expose_api, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class GroupAPIController(BaseAPIController): - - def __init__(self, app: StructuredApp): - super().__init__(app) - self.manager = GroupsManager(app) +class GroupAPIController(BaseGalaxyAPIController): + manager = depends(GroupsManager) @expose_api @require_admin diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 701542b4935..f299011bc19 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -34,27 +34,24 @@ from galaxy.web import ( expose_api_raw, ) from galaxy.webapps.base.controller import ( - BaseAPIController, ExportsHistoryMixin, ImportsHistoryMixin, SharableMixin ) +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class HistoriesController(BaseAPIController, ExportsHistoryMixin, ImportsHistoryMixin, SharableMixin): - - def __init__(self, app): - super().__init__(app) - self.citations_manager = citations.CitationsManager(app) - self.user_manager = users.UserManager(app) - self.workflow_manager = workflows.WorkflowsManager(app) - self.manager = histories.HistoryManager(app) - self.history_export_view = histories.HistoryExportView(app) - self.serializer = histories.HistorySerializer(app) - self.deserializer = histories.HistoryDeserializer(app) - self.filters = histories.HistoryFilters(app) +class HistoriesController(BaseGalaxyAPIController, ExportsHistoryMixin, ImportsHistoryMixin, SharableMixin): + citations_manager: citations.CitationsManager = depends(citations.CitationsManager) + user_manager: users.UserManager = depends(users.UserManager) + workflow_manager: workflows.WorkflowsManager = depends(workflows.WorkflowsManager) + manager: histories.HistoryManager = depends(histories.HistoryManager) + history_export_view: histories.HistoryExportView = depends(histories.HistoryExportView) + serializer: histories.HistorySerializer = depends(histories.HistorySerializer) + deserializer: histories.HistoryDeserializer = depends(histories.HistoryDeserializer) + filters: histories.HistoryFilters = depends(histories.HistoryFilters) @expose_api_anonymous def index(self, trans, deleted='False', **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index eec95949649..2f93ae00360 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -30,27 +30,24 @@ from galaxy.web import ( expose_api_raw_anonymous ) from galaxy.webapps.base.controller import ( - BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems, UsesTagsMixin ) +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems, UsesTagsMixin): - - def __init__(self, app): - super().__init__(app) - self.hda_manager = hdas.HDAManager(app) - self.history_manager = histories.HistoryManager(app) - self.history_contents_manager = history_contents.HistoryContentsManager(app) - self.folder_manager = folders.FolderManager() - self.hda_serializer = hdas.HDASerializer(app) - self.hda_deserializer = hdas.HDADeserializer(app) - self.hdca_serializer = hdcas.HDCASerializer(app) - self.history_contents_filters = history_contents.HistoryContentsFilters(app) +class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixin, UsesLibraryMixinItems, UsesTagsMixin): + hda_manager: hdas.HDAManager = depends(hdas.HDAManager) + history_manager: histories.HistoryManager = depends(histories.HistoryManager) + history_contents_manager: history_contents.HistoryContentsManager = depends(history_contents.HistoryContentsManager) + folder_manager: folders.FolderManager = depends(folders.FolderManager) + hda_serializer: hdas.HDASerializer = depends(hdas.HDASerializer) + hda_deserializer: hdas.HDADeserializer = depends(hdas.HDADeserializer) + hdca_serializer: hdcas.HDCASerializer = depends(hdcas.HDCASerializer) + history_contents_filters: history_contents.HistoryContentsFilters = depends(history_contents.HistoryContentsFilters) @expose_api_anonymous def index(self, trans, history_id, ids=None, v=None, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/item_tags.py b/lib/galaxy/webapps/galaxy/api/item_tags.py index c10a5557c4a..63fae9ddb21 100644 --- a/lib/galaxy/webapps/galaxy/api/item_tags.py +++ b/lib/galaxy/webapps/galaxy/api/item_tags.py @@ -6,14 +6,14 @@ import logging from galaxy import exceptions from galaxy.web import expose_api from galaxy.webapps.base.controller import ( - BaseAPIController, UsesTagsMixin ) +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class BaseItemTagsController(BaseAPIController, UsesTagsMixin): +class BaseItemTagsController(BaseGalaxyAPIController, UsesTagsMixin): """ """ @expose_api diff --git a/lib/galaxy/webapps/galaxy/api/job_files.py b/lib/galaxy/webapps/galaxy/api/job_files.py index b3bd3b2972c..282af0249b2 100644 --- a/lib/galaxy/webapps/galaxy/api/job_files.py +++ b/lib/galaxy/webapps/galaxy/api/job_files.py @@ -14,12 +14,12 @@ from galaxy.web import ( expose_api_anonymous_and_sessionless, expose_api_raw_anonymous_and_sessionless, ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class JobFilesAPIController(BaseAPIController): +class JobFilesAPIController(BaseGalaxyAPIController): """ This job files controller allows remote job running mechanisms to read and modify the current state of files for queued and running jobs. It is certainly not meant to represent part of Galaxy's stable, user diff --git a/lib/galaxy/webapps/galaxy/api/job_lock.py b/lib/galaxy/webapps/galaxy/api/job_lock.py index f64e8809c8a..18e68515171 100644 --- a/lib/galaxy/webapps/galaxy/api/job_lock.py +++ b/lib/galaxy/webapps/galaxy/api/job_lock.py @@ -1,22 +1,21 @@ from fastapi import ( Body, - Depends, ) from fastapi_utils.inferring_router import InferringRouter as APIRouter -from galaxy.managers.jobs import JobLock -from . import get_admin_user, get_job_manager +from galaxy.managers.jobs import JobLock, JobManager +from . import AdminUserRequired, depends router = APIRouter(tags=['job_lock']) -@router.get('/api/job_lock') -def job_lock_status(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user)) -> JobLock: +@router.get('/api/job_lock', dependencies=[AdminUserRequired]) +def job_lock_status(job_manager: JobManager = depends(JobManager)) -> JobLock: """Get job lock status.""" return job_manager.job_lock() -@router.put('/api/job_lock') -def update_job_lock(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user), job_lock: JobLock = Body(...)) -> JobLock: +@router.put('/api/job_lock', dependencies=[AdminUserRequired]) +def update_job_lock(job_manager: JobManager = depends(JobManager), job_lock: JobLock = Body(...)) -> JobLock: """Set job lock status.""" return job_manager.update_job_lock(job_lock) diff --git a/lib/galaxy/webapps/galaxy/api/job_ports.py b/lib/galaxy/webapps/galaxy/api/job_ports.py index 0b4ec80177d..76054cda133 100644 --- a/lib/galaxy/webapps/galaxy/api/job_ports.py +++ b/lib/galaxy/webapps/galaxy/api/job_ports.py @@ -2,11 +2,12 @@ related to running and queued jobs. """ from galaxy.job_execution.ports import JobPortsView +from galaxy.structured_app import StructuredApp from galaxy.web import expose_api_anonymous_and_sessionless -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController -class JobPortsAPIController(BaseAPIController): +class JobPortsAPIController(BaseGalaxyAPIController): """ This job files controller allows remote job running mechanisms to modify the current state of ports for queued and running jobs. It is certainly not meant to represent part of Galaxy's stable, user @@ -15,7 +16,8 @@ class JobPortsAPIController(BaseAPIController): See the JobFiles API for information about per-job API keys. """ - def __init__(self, app): + def __init__(self, app: StructuredApp): + super().__init__(app) self._job_ports_view = JobPortsView(app) @expose_api_anonymous_and_sessionless diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index dc56cddb868..1eb6627687a 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -7,7 +7,6 @@ API operations on a jobs. import logging import typing -from fastapi import Depends from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter from sqlalchemy import ( @@ -19,7 +18,6 @@ from galaxy import ( model, util, ) -from galaxy.app import UniverseApplication from galaxy.managers import hdas from galaxy.managers.context import ProvidesHistoryContext, ProvidesUserContext from galaxy.managers.jobs import ( @@ -39,16 +37,15 @@ from galaxy.web import ( require_admin, ) from galaxy.webapps.base.controller import ( - BaseAPIController, UsesVisualizationMixin ) from galaxy.work.context import ( WorkRequestContext, ) from . import ( - get_app, - get_job_manager, - get_trans, + BaseGalaxyAPIController, + depends, + DependsOnTrans, ) log = logging.getLogger(__name__) @@ -56,22 +53,14 @@ log = logging.getLogger(__name__) router = APIRouter(tags=["jobs"]) -def get_job_search(app: UniverseApplication = Depends(get_app)) -> JobSearch: - return JobSearch(app=app) - - -def get_hda_manager(app: UniverseApplication = Depends(get_app)) -> hdas.HDAManager: - return app.hda_manager - - @cbv(router) class FastAPIJobs: - job_manager: JobManager = Depends(get_job_manager) - job_search: JobSearch = Depends(get_job_search) - hda_manager: hdas.HDAManager = Depends(get_hda_manager) + job_manager: JobManager = depends(JobManager) + job_search: JobSearch = depends(JobSearch) + hda_manager: hdas.HDAManager = depends(hdas.HDAManager) @router.get("/api/job/{id}") - def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans), full: typing.Optional[bool] = False) -> typing.Dict: + def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = DependsOnTrans, full: typing.Optional[bool] = False) -> typing.Dict: """ Return dictionary containing description of job data @@ -84,13 +73,10 @@ class FastAPIJobs: return view_show_job(trans, job, bool(full)) -class JobController(BaseAPIController, UsesVisualizationMixin): - - def __init__(self, app): - super().__init__(app) - self.job_manager = JobManager(app) - self.job_search = JobSearch(app) - self.hda_manager = hdas.HDAManager(app) +class JobController(BaseGalaxyAPIController, UsesVisualizationMixin): + job_manager = depends(JobManager) + job_search = depends(JobSearch) + hda_manager = depends(hdas.HDAManager) @expose_api def index(self, trans: ProvidesUserContext, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/libraries.py b/lib/galaxy/webapps/galaxy/api/libraries.py index 8417bed059f..90768f66783 100644 --- a/lib/galaxy/webapps/galaxy/api/libraries.py +++ b/lib/galaxy/webapps/galaxy/api/libraries.py @@ -13,16 +13,13 @@ from galaxy.web import ( expose_api, expose_api_anonymous, ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class LibrariesController(BaseAPIController): - - def __init__(self, app): - super().__init__(app) - self.manager = libraries.LibrariesManager(app) +class LibrariesController(BaseGalaxyAPIController): + manager = depends(libraries.LibrariesManager) @expose_api_anonymous def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/library_contents.py b/lib/galaxy/webapps/galaxy/api/library_contents.py index a5d894e9f03..7237cfd4ee7 100644 --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -23,23 +23,25 @@ from galaxy.model import ( ExtendedMetadataIndex, tags ) +from galaxy.structured_app import StructuredApp from galaxy.web import expose_api from galaxy.webapps.base.controller import ( - BaseAPIController, HTTPBadRequest, url_for, UsesFormDefinitionsMixin, UsesLibraryMixin, UsesLibraryMixinItems ) +from . import BaseGalaxyAPIController + log = logging.getLogger(__name__) -class LibraryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems, UsesFormDefinitionsMixin, LibraryActions): +class LibraryContentsController(BaseGalaxyAPIController, UsesLibraryMixin, UsesLibraryMixinItems, UsesFormDefinitionsMixin, LibraryActions): - def __init__(self, app): + def __init__(self, app: StructuredApp, hda_manager: managers.hdas.HDAManager): super().__init__(app) - self.hda_manager = managers.hdas.HDAManager(app) + self.hda_manager = hda_manager @expose_api def index(self, trans, library_id, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/library_datasets.py b/lib/galaxy/webapps/galaxy/api/library_datasets.py index dfe82e65437..624b9744614 100644 --- a/lib/galaxy/webapps/galaxy/api/library_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/library_datasets.py @@ -22,6 +22,7 @@ from galaxy.managers import ( library_datasets, roles ) +from galaxy.structured_app import StructuredApp from galaxy.tools.actions import upload_common from galaxy.tools.parameters import populate_state from galaxy.util.path import ( @@ -36,16 +37,16 @@ from galaxy.web import ( expose_api_anonymous, ) from galaxy.webapps.base.controller import ( - BaseAPIController, UsesVisualizationMixin, ) +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, LibraryActions): +class LibraryDatasetsController(BaseGalaxyAPIController, UsesVisualizationMixin, LibraryActions): - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.app = app self.folder_manager = folders.FolderManager() diff --git a/lib/galaxy/webapps/galaxy/api/licenses.py b/lib/galaxy/webapps/galaxy/api/licenses.py index e7e6f09df93..73bc66e6c0a 100644 --- a/lib/galaxy/webapps/galaxy/api/licenses.py +++ b/lib/galaxy/webapps/galaxy/api/licenses.py @@ -1,7 +1,6 @@ from typing import List from fastapi import ( - Depends, Path ) from fastapi_utils.cbv import cbv @@ -12,7 +11,10 @@ from galaxy.managers.licenses import ( LicensesManager ) from galaxy.web import expose_api_anonymous_and_sessionless -from galaxy.webapps.base.controller import BaseAPIController +from . import ( + BaseGalaxyAPIController, + depends, +) router = APIRouter(tags=['licenses']) @@ -24,13 +26,9 @@ LicenseIdPath: str = Path( ) -def get_licenses_manager() -> LicensesManager: - return LicensesManager() - - @cbv(router) class FastAPILicenses: - licenses_manager: LicensesManager = Depends(get_licenses_manager) + licenses_manager: LicensesManager = depends(LicensesManager) @router.get('/api/licenses', summary="Lists all available SPDX licenses", @@ -48,10 +46,8 @@ class FastAPILicenses: return self.licenses_manager.get_license_by_id(id) -class LicensesController(BaseAPIController): - - def __init__(self, app): - self.licenses_manager = LicensesManager() +class LicensesController(BaseGalaxyAPIController): + licenses_manager: LicensesManager = depends(LicensesManager) @expose_api_anonymous_and_sessionless def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/metrics.py b/lib/galaxy/webapps/galaxy/api/metrics.py index 699551a7d1e..4caa2809150 100644 --- a/lib/galaxy/webapps/galaxy/api/metrics.py +++ b/lib/galaxy/webapps/galaxy/api/metrics.py @@ -7,15 +7,16 @@ API operations for for querying and recording user metrics from some client import datetime import logging +from galaxy.structured_app import StructuredApp from galaxy.web import expose_api_anonymous -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class MetricsController(BaseAPIController): +class MetricsController(BaseGalaxyAPIController): - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) #: set to true to send additional debugging info to the log self.debugging = True diff --git a/lib/galaxy/webapps/galaxy/api/page_revisions.py b/lib/galaxy/webapps/galaxy/api/page_revisions.py index d1b793b31ca..e8cebb69c41 100644 --- a/lib/galaxy/webapps/galaxy/api/page_revisions.py +++ b/lib/galaxy/webapps/galaxy/api/page_revisions.py @@ -8,19 +8,16 @@ from galaxy.managers.pages import PageManager from galaxy.model.item_attrs import UsesAnnotations from galaxy.web import expose_api from galaxy.webapps.base.controller import ( - BaseAPIController, SharableItemSecurityMixin, SharableMixin ) +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotations, SharableMixin): - - def __init__(self, app): - super().__init__(app) - self.manager = PageManager(app) +class PageRevisionsController(BaseGalaxyAPIController, SharableItemSecurityMixin, UsesAnnotations, SharableMixin): + manager: PageManager = depends(PageManager) @expose_api def index(self, trans, page_id, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/pages.py b/lib/galaxy/webapps/galaxy/api/pages.py index a50513e080e..8ea7a3bc427 100644 --- a/lib/galaxy/webapps/galaxy/api/pages.py +++ b/lib/galaxy/webapps/galaxy/api/pages.py @@ -6,7 +6,6 @@ import logging from fastapi import ( Body, - Depends, Path, Query, status, @@ -17,7 +16,9 @@ from starlette.responses import StreamingResponse from galaxy.managers.context import ProvidesUserContext from galaxy.managers.pages import ( - CreatePagePayload, PageDetails, + CreatePagePayload, + PageDetails, + PageSerializer, PagesManager, PageSummary, PageSummaryList, @@ -28,8 +29,7 @@ from galaxy.web import ( expose_api_anonymous_and_sessionless, expose_api_raw_anonymous_and_sessionless ) -from galaxy.webapps.base.controller import BaseAPIController -from . import get_app, get_trans +from . import BaseGalaxyAPIController, depends, DependsOnTrans log = logging.getLogger(__name__) @@ -48,13 +48,9 @@ PageIdPathParam: EncodedDatabaseIdField = Path( ) -def get_pages_manager(app=Depends(get_app)) -> PagesManager: - return PagesManager(app) - - @cbv(router) class FastAPIPages: - manager: PagesManager = Depends(get_pages_manager) + manager: PagesManager = depends(PagesManager) @router.get( '/api/pages', @@ -63,7 +59,7 @@ class FastAPIPages: ) async def index( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, deleted: bool = DeletedQueryParam, ) -> PageSummaryList: """Get a list with summary information of all Pages available to the user.""" @@ -76,7 +72,7 @@ class FastAPIPages: ) def create( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, payload: CreatePagePayload = Body(...), ) -> PageSummary: """Get a list with details of all Pages available to the user.""" @@ -89,7 +85,7 @@ class FastAPIPages: ) async def delete( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, id: EncodedDatabaseIdField = PageIdPathParam, ): """Marks the Page with the given ID as deleted.""" @@ -102,7 +98,7 @@ class FastAPIPages: ) async def show( self, - trans=Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, id: EncodedDatabaseIdField = PageIdPathParam, ) -> PageDetails: """Return summary information about a specific Page and the content of the last revision.""" @@ -124,7 +120,7 @@ class FastAPIPages: ) async def show_pdf( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, id: EncodedDatabaseIdField = PageIdPathParam, ): """Return a PDF document of the last revision of the Page. @@ -135,14 +131,12 @@ class FastAPIPages: return StreamingResponse(io.BytesIO(pdf_bytes), media_type="application/pdf") -class PagesController(BaseAPIController): +class PagesController(BaseGalaxyAPIController): """ RESTful controller for interactions with pages. """ - - def __init__(self, app): - super().__init__(app) - self.manager = PagesManager(app) + manager: PagesManager = depends(PagesManager) + serializer: PageSerializer = depends(PageSerializer) @expose_api_anonymous_and_sessionless def index(self, trans, deleted=False, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/plugins.py b/lib/galaxy/webapps/galaxy/api/plugins.py index 7da94c5110a..e69276a96fb 100644 --- a/lib/galaxy/webapps/galaxy/api/plugins.py +++ b/lib/galaxy/webapps/galaxy/api/plugins.py @@ -7,20 +7,17 @@ from galaxy import exceptions from galaxy.managers import hdas, histories from galaxy.util import asbool from galaxy.web import expose_api -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class PluginsController(BaseAPIController): +class PluginsController(BaseGalaxyAPIController): """ RESTful controller for interactions with plugins. """ - - def __init__(self, app): - super().__init__(app) - self.hda_manager = hdas.HDAManager(app) - self.history_manager = histories.HistoryManager(app) + hda_manager: hdas.HDAManager = depends(hdas.HDAManager) + history_manager: histories.HistoryManager = depends(histories.HistoryManager) @expose_api def index(self, trans, **kwargs): diff --git a/lib/galaxy/webapps/galaxy/api/provenance.py b/lib/galaxy/webapps/galaxy/api/provenance.py index b03182a926c..187db11fff3 100644 --- a/lib/galaxy/webapps/galaxy/api/provenance.py +++ b/lib/galaxy/webapps/galaxy/api/provenance.py @@ -9,22 +9,18 @@ from paste.httpexceptions import ( ) from galaxy import ( - managers, web ) -from galaxy.webapps.base.controller import BaseAPIController +from galaxy.managers.hdas import HDAManager +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class BaseProvenanceController(BaseAPIController): +class BaseProvenanceController(BaseGalaxyAPIController): """ """ - def __init__(self, app): - super().__init__(app) - self.hda_manager = managers.hdas.HDAManager(app) - @web.legacy_expose_api def index(self, trans, **kwd): follow = kwd.get('follow', False) @@ -98,9 +94,11 @@ class HDAProvenanceController(BaseProvenanceController): controller_name = "history_content_provenance" provenance_item_class = "HistoryDatasetAssociation" provenance_item_id = "history_content_id" + hda_manager: HDAManager = depends(HDAManager) class LDDAProvenanceController(BaseProvenanceController): controller_name = "ldda_provenance" provenance_item_class = "LibraryDatasetDatasetAssociation" provenance_item_id = "library_content_id" + hda_manager: HDAManager = depends(HDAManager) diff --git a/lib/galaxy/webapps/galaxy/api/quotas.py b/lib/galaxy/webapps/galaxy/api/quotas.py index cdcab3729e3..b54db896939 100644 --- a/lib/galaxy/webapps/galaxy/api/quotas.py +++ b/lib/galaxy/webapps/galaxy/api/quotas.py @@ -17,15 +17,16 @@ from galaxy.actions.admin import AdminActions from galaxy.exceptions import ActionInputError from galaxy.web.params import QuotaParamParser from galaxy.webapps.base.controller import ( - BaseAPIController, url_for, UsesQuotaMixin ) +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class QuotaAPIController(BaseAPIController, AdminActions, UsesQuotaMixin, QuotaParamParser): +class QuotaAPIController(BaseGalaxyAPIController, AdminActions, UsesQuotaMixin, QuotaParamParser): + @web.require_admin @web.legacy_expose_api def index(self, trans, deleted='False', **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/remote_files.py b/lib/galaxy/webapps/galaxy/api/remote_files.py index fedb777e338..5dc5e836c39 100644 --- a/lib/galaxy/webapps/galaxy/api/remote_files.py +++ b/lib/galaxy/webapps/galaxy/api/remote_files.py @@ -4,7 +4,6 @@ API operations on remote files. import logging from typing import Any, Dict, List, Optional -from fastapi import Depends from fastapi.param_functions import Query from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter @@ -17,12 +16,11 @@ from galaxy.files._schema import ( ) from galaxy.managers.context import ProvidesUserContext from galaxy.managers.remote_files import RemoteFilesManager -from galaxy.structured_app import StructuredApp from galaxy.web import expose_api -from galaxy.webapps.base.controller import BaseAPIController from . import ( - get_app, - get_trans, + BaseGalaxyAPIController, + depends, + DependsOnTrans, ) log = logging.getLogger(__name__) @@ -67,13 +65,9 @@ DisableModeQueryParam: Optional[RemoteFilesDisableMode] = Query( ) -def get_remote_files_manager(app: StructuredApp = Depends(get_app)) -> RemoteFilesManager: - return RemoteFilesManager(app) # TODO: remove/refactor after merging #11180 - - @cbv(router) class FastAPIRemoteFiles: - manager: RemoteFilesManager = Depends(get_remote_files_manager) + manager: RemoteFilesManager = depends(RemoteFilesManager) @router.get( '/api/remote_files', @@ -82,7 +76,7 @@ class FastAPIRemoteFiles: ) async def index( self, - user_ctx: ProvidesUserContext = Depends(get_trans), + user_ctx: ProvidesUserContext = DependsOnTrans, target: str = TargetQueryParam, format: Optional[RemoteFilesFormat] = FormatQueryParam, recursive: Optional[bool] = RecursiveQueryParam, @@ -98,17 +92,14 @@ class FastAPIRemoteFiles: ) async def plugins( self, - user_ctx: ProvidesUserContext = Depends(get_trans), + user_ctx: ProvidesUserContext = DependsOnTrans, ) -> FilesSourcePluginList: """Display plugin information for each of the gxfiles:// URI targets available.""" return self.manager.get_files_source_plugins() -class RemoteFilesAPIController(BaseAPIController): - - def __init__(self, app): - super().__init__(app) - self.manager = RemoteFilesManager(app) +class RemoteFilesAPIController(BaseGalaxyAPIController): + manager: RemoteFilesManager = depends(RemoteFilesManager) @expose_api def index(self, trans: ProvidesUserContext, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/roles.py b/lib/galaxy/webapps/galaxy/api/roles.py index b320fb8bd4c..9f653ba6148 100644 --- a/lib/galaxy/webapps/galaxy/api/roles.py +++ b/lib/galaxy/webapps/galaxy/api/roles.py @@ -7,7 +7,6 @@ from typing import List from fastapi import ( Body, - Depends, ) from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter @@ -16,7 +15,6 @@ from pydantic import ( ) from galaxy import web -from galaxy.app import UniverseApplication from galaxy.managers.base import decode_id from galaxy.managers.context import ProvidesUserContext from galaxy.managers.roles import ( @@ -25,11 +23,12 @@ from galaxy.managers.roles import ( RoleModel, ) from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.webapps.base.controller import BaseAPIController, url_for +from galaxy.webapps.base.controller import url_for from . import ( - get_admin_user, - get_app, - get_trans, + AdminUserRequired, + BaseGalaxyAPIController, + depends, + DependsOnTrans, ) log = logging.getLogger(__name__) @@ -54,32 +53,29 @@ def role_to_model(trans, role): return RoleModel(**item) -def get_role_manager(app: UniverseApplication = Depends(get_app)) -> RoleManager: - return app.role_manager - - @cbv(router) class FastAPIRoles: - role_manager: RoleManager = Depends(get_role_manager) + role_manager: RoleManager = depends(RoleManager) @router.get('/api/roles') - def index(self, trans: ProvidesUserContext = Depends(get_trans)) -> RoleListModel: + def index(self, trans: ProvidesUserContext = DependsOnTrans) -> RoleListModel: roles = self.role_manager.list_displayable_roles(trans) return RoleListModel(__root__=[role_to_model(trans, r) for r in roles]) @router.get('/api/roles/{id}') - def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans)) -> RoleModel: + def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = DependsOnTrans) -> RoleModel: role_id = trans.app.security.decode_id(id) role = self.role_manager.get(trans, role_id) return role_to_model(trans, role) - @router.post("/api/roles") - def create(self, trans: ProvidesUserContext = Depends(get_trans), admin_user=Depends(get_admin_user), role_definition_model: RoleDefinitionModel = Body(...)) -> RoleModel: + @router.post("/api/roles", dependencies=[AdminUserRequired]) + def create(self, trans: ProvidesUserContext = DependsOnTrans, role_definition_model: RoleDefinitionModel = Body(...)) -> RoleModel: role = self.role_manager.create_role(trans, role_definition_model) return role_to_model(trans, role) -class RoleAPIController(BaseAPIController): +class RoleAPIController(BaseGalaxyAPIController): + role_manager: RoleManager = depends(RoleManager) @web.expose_api def index(self, trans: ProvidesUserContext, **kwd): @@ -87,7 +83,7 @@ class RoleAPIController(BaseAPIController): GET /api/roles Displays a collection (list) of roles. """ - roles = self._role_manager.list_displayable_roles(trans) + roles = self.role_manager.list_displayable_roles(trans) return RoleListModel(__root__=[role_to_model(trans, r) for r in roles]) @web.expose_api @@ -97,7 +93,7 @@ class RoleAPIController(BaseAPIController): Displays information about a role. """ role_id = decode_id(self.app, id) - role = self._role_manager.get(trans, role_id) + role = self.role_manager.get(trans, role_id) return role_to_model(trans, role) @web.expose_api @@ -109,13 +105,9 @@ class RoleAPIController(BaseAPIController): """ expand_json_keys(payload, ["user_ids", "group_ids"]) role_definition_model = RoleDefinitionModel(**payload) - role = self._role_manager.create_role(trans, role_definition_model) + role = self.role_manager.create_role(trans, role_definition_model) return role_to_model(trans, role) - @property - def _role_manager(self): - return self.app.role_manager - def expand_json_keys(payload: dict, keys): for key in keys: diff --git a/lib/galaxy/webapps/galaxy/api/search.py b/lib/galaxy/webapps/galaxy/api/search.py index 3b2871069ff..a12be20b22c 100644 --- a/lib/galaxy/webapps/galaxy/api/search.py +++ b/lib/galaxy/webapps/galaxy/api/search.py @@ -8,15 +8,13 @@ from galaxy.exceptions import ItemAccessibilityException from galaxy.managers.context import ProvidesUserContext from galaxy.model.search import GalaxySearchEngine from galaxy.util import unicodify -from galaxy.webapps.base.controller import ( - BaseAPIController, - SharableItemSecurityMixin -) +from galaxy.webapps.base.controller import SharableItemSecurityMixin +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class SearchController(BaseAPIController, SharableItemSecurityMixin): +class SearchController(BaseGalaxyAPIController, SharableItemSecurityMixin): @web.legacy_expose_api def create(self, trans: ProvidesUserContext, payload: dict, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/tags.py b/lib/galaxy/webapps/galaxy/api/tags.py index 9af7182d565..b17962a9240 100644 --- a/lib/galaxy/webapps/galaxy/api/tags.py +++ b/lib/galaxy/webapps/galaxy/api/tags.py @@ -17,10 +17,10 @@ from galaxy.managers.tags import ( TagsManager, ) from galaxy.web import expose_api -from galaxy.webapps.base.controller import BaseAPIController from . import ( - Depends, - get_trans, + BaseGalaxyAPIController, + depends, + DependsOnTrans, ) log = logging.getLogger(__name__) @@ -28,13 +28,9 @@ log = logging.getLogger(__name__) router = APIRouter(tags=['tags']) -def get_tags_manager() -> TagsManager: - return TagsManager() # TODO: remove/refactor after merging #11180 - - @cbv(router) class FastAPITags: - manager: TagsManager = Depends(get_tags_manager) + manager: TagsManager = depends(TagsManager) @router.put( '/api/tags', @@ -43,7 +39,7 @@ class FastAPITags: ) def update( self, - trans: ProvidesUserContext = Depends(get_trans), + trans: ProvidesUserContext = DependsOnTrans, payload: ItemTagsPayload = Body( ..., # Required title="Payload", @@ -58,11 +54,8 @@ class FastAPITags: self.manager.update(trans, payload) -class TagsController(BaseAPIController): - - def __init__(self, app): - super().__init__(app) - self.manager = TagsManager() +class TagsController(BaseGalaxyAPIController): + manager: TagsManager = depends(TagsManager) # Retag an item. All previous tags are deleted and new tags are applied. @expose_api diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py index d4a28781e1b..62ccbe92015 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py +++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py @@ -5,19 +5,20 @@ import logging from typing import Optional from galaxy.managers.context import ProvidesAppContext +from galaxy.structured_app import StructuredApp from galaxy.tool_util.deps import views from galaxy.web import ( expose_api, require_admin ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class ToolDependenciesAPIController(BaseAPIController): +class ToolDependenciesAPIController(BaseGalaxyAPIController): - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self._view = views.DependencyResolversView(app) diff --git a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py index 6d792cabc08..d339c95212e 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py +++ b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py @@ -9,15 +9,16 @@ from galaxy.model import ( InteractiveToolEntryPoint, Job, ) +from galaxy.structured_app import StructuredApp from galaxy.web import expose_api_anonymous_and_sessionless -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class ToolEntryPointsAPIController(BaseAPIController): +class ToolEntryPointsAPIController(BaseGalaxyAPIController): - def __init__(self, app): + def __init__(self, app: StructuredApp): self.app = app self.interactivetool_manager = app.interactivetool_manager diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index ade2cad3315..aed41af1b72 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -34,7 +34,7 @@ from galaxy.web import ( require_admin, url_for ) -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def get_message_for_no_shed_tool_config(): return message -class ToolShedRepositoriesController(BaseAPIController): +class ToolShedRepositoriesController(BaseGalaxyAPIController): """RESTful controller for interactions with tool shed repositories.""" def __ensure_can_install_repos(self, trans): diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 99a6a7e1211..fa0ba60e8ad 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -5,6 +5,8 @@ from typing import Any, cast, Dict, Optional from galaxy import exceptions, util, web from galaxy.managers.collections_util import dictify_dataset_collection_instance +from galaxy.managers.hdas import HDAManager +from galaxy.managers.histories import HistoryManager from galaxy.model import PostJobAction from galaxy.tools import global_tool_errors from galaxy.util.zipstream import ZipstreamWrapper @@ -14,9 +16,9 @@ from galaxy.web import ( expose_api_anonymous_and_sessionless, expose_api_raw_anonymous_and_sessionless, ) -from galaxy.webapps.base.controller import BaseAPIController from galaxy.webapps.base.controller import UsesVisualizationMixin from galaxy.webapps.base.webapp import GalaxyWebTransaction +from . import BaseGalaxyAPIController, depends from ._fetch_util import validate_and_normalize_targets log = logging.getLogger(__name__) @@ -28,15 +30,12 @@ PROTECTED_TOOLS = ["__DATA_FETCH__"] SEARCH_RESERVED_TERMS_FAVORITES = ['#favs', '#favorites', '#favourites'] -class ToolsController(BaseAPIController, UsesVisualizationMixin): +class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin): """ RESTful controller for interactions with tools. """ - - def __init__(self, app): - super().__init__(app) - self.history_manager = app.history_manager - self.hda_manager = app.hda_manager + history_manager: HistoryManager = depends(HistoryManager) + hda_manager: HDAManager = depends(HDAManager) @expose_api_anonymous_and_sessionless def index(self, trans: GalaxyWebTransaction, **kwds): diff --git a/lib/galaxy/webapps/galaxy/api/toolshed.py b/lib/galaxy/webapps/galaxy/api/toolshed.py index 696e028f58b..e2d1d798773 100644 --- a/lib/galaxy/webapps/galaxy/api/toolshed.py +++ b/lib/galaxy/webapps/galaxy/api/toolshed.py @@ -5,12 +5,12 @@ from urllib.parse import quote from galaxy.exceptions import MessageException from galaxy.util import url_get from galaxy.web import expose_api, require_admin -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class ToolShedController(BaseAPIController): +class ToolShedController(BaseGalaxyAPIController): """RESTful controller for interactions with Toolsheds.""" @expose_api diff --git a/lib/galaxy/webapps/galaxy/api/tours.py b/lib/galaxy/webapps/galaxy/api/tours.py index dfb55b9ea66..54525421edf 100644 --- a/lib/galaxy/webapps/galaxy/api/tours.py +++ b/lib/galaxy/webapps/galaxy/api/tours.py @@ -3,7 +3,6 @@ API Controller providing Galaxy Tours """ import logging -from fastapi import Depends from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter @@ -17,10 +16,10 @@ from galaxy.web import ( expose_api_anonymous_and_sessionless, require_admin ) -from galaxy.webapps.base.controller import BaseAPIController from . import ( - get_admin_user, - get_app, + AdminUserRequired, + BaseGalaxyAPIController, + depends, ) log = logging.getLogger(__name__) @@ -29,13 +28,10 @@ log = logging.getLogger(__name__) router = APIRouter(tags=['tours']) -def get_tours_registry(app=Depends(get_app)) -> ToursRegistry: - return app.tour_registry - - @cbv(router) class FastAPITours: - registry: ToursRegistry = Depends(get_tours_registry) + # ugh - mypy https://github.com/python/mypy/issues/5374 + registry: ToursRegistry = depends(ToursRegistry) # type: ignore @router.get('/api/tours') def index(self) -> TourList: @@ -47,16 +43,14 @@ class FastAPITours: """Return a tour definition.""" return self.registry.tour_contents(tour_id) - @router.post('/api/tours/{tour_id}', dependencies=[Depends(get_admin_user)]) + @router.post('/api/tours/{tour_id}', dependencies=[AdminUserRequired]) def update_tour(self, tour_id: str) -> TourDetails: """Return a tour definition.""" return self.registry.load_tour(tour_id) -class ToursController(BaseAPIController): - - def __init__(self, app): - super().__init__(app) +class ToursController(BaseGalaxyAPIController): + registry: ToursRegistry = depends(ToursRegistry) # type: ignore @expose_api_anonymous_and_sessionless def index(self, trans, **kwd): @@ -65,7 +59,7 @@ class ToursController(BaseAPIController): Displays available tours """ - return self.app.tour_registry.get_tours() + return self.registry.get_tours() @expose_api_anonymous_and_sessionless def show(self, trans, tour_id, **kwd): @@ -77,7 +71,7 @@ class ToursController(BaseAPIController): :returns: tour definition :rtype: dictionary """ - return self.app.tour_registry.tour_contents(tour_id) + return self.registry.tour_contents(tour_id) @require_admin @expose_api @@ -85,4 +79,4 @@ class ToursController(BaseAPIController): """This simply reloads tours right now. It's a quick hack.""" # TODO: allow creation of new tours (which get written to the # filesystem). - return self.app.tour_registry.load_tour(tour_id) + return self.registry.load_tour(tour_id) diff --git a/lib/galaxy/webapps/galaxy/api/trs_consumer.py b/lib/galaxy/webapps/galaxy/api/trs_consumer.py index 910e94cefbc..3e0043fbd8c 100644 --- a/lib/galaxy/webapps/galaxy/api/trs_consumer.py +++ b/lib/galaxy/webapps/galaxy/api/trs_consumer.py @@ -3,16 +3,13 @@ Information on TRS can be found at https://github.com/ga4gh/tool-registry-service-schemas. """ from galaxy.web import expose_api -from galaxy.webapps.base.controller import ( - BaseAPIController -) +from galaxy.workflow.trs_proxy import TrsProxy +from . import BaseGalaxyAPIController, depends -class TrsConsumeAPIController(BaseAPIController): +class TrsConsumeAPIController(BaseGalaxyAPIController): """Controller for TRS proxying.""" - - def __init__(self, app): - self._trs_proxy = app.trs_proxy + _trs_proxy: TrsProxy = depends(TrsProxy) @expose_api def get_servers(self, trans, *args, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/trs_search.py b/lib/galaxy/webapps/galaxy/api/trs_search.py index 666b017444d..ce2db31f4ae 100644 --- a/lib/galaxy/webapps/galaxy/api/trs_search.py +++ b/lib/galaxy/webapps/galaxy/api/trs_search.py @@ -5,23 +5,19 @@ Information on TRS can be found at https://github.com/ga4gh/tool-registry-servic import logging from galaxy.web import expose_api -from galaxy.webapps.base.controller import ( - BaseAPIController -) -from galaxy.workflow.trs_proxy import parse_search_kwds +from galaxy.workflow.trs_proxy import parse_search_kwds, TrsProxy +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class TrsSearchAPIController(BaseAPIController): +class TrsSearchAPIController(BaseGalaxyAPIController): """Controller for TRS searching. Not trying to emulate the actual underlying GA4GH API interface so throwing the search functionality into a different controller than the trs_consumer. """ - - def __init__(self, app): - self._trs_proxy = app.trs_proxy + _trs_proxy: TrsProxy = depends(TrsProxy) @expose_api def index(self, trans, trs_server=None, query=None, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/uploads.py b/lib/galaxy/webapps/galaxy/api/uploads.py index 1519fd8f33f..c3226261be4 100644 --- a/lib/galaxy/webapps/galaxy/api/uploads.py +++ b/lib/galaxy/webapps/galaxy/api/uploads.py @@ -7,12 +7,12 @@ import re from galaxy import exceptions from galaxy.web import legacy_expose_api_anonymous -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class UploadsAPIController(BaseAPIController): +class UploadsAPIController(BaseGalaxyAPIController): READ_CHUNK_SIZE = 2 ** 16 diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 70b0afbc980..75f19224f03 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -41,25 +41,21 @@ from galaxy.web import ( ) from galaxy.web.form_builder import AddressField from galaxy.webapps.base.controller import ( - BaseAPIController, BaseUIController, UsesFormDefinitionsMixin, UsesTagsMixin ) from galaxy.webapps.base.webapp import GalaxyWebTransaction - +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class UserAPIController(BaseAPIController, UsesTagsMixin, BaseUIController, UsesFormDefinitionsMixin): - - def __init__(self, app): - super().__init__(app) - self.user_manager = users.UserManager(app) - self.user_serializer = users.UserSerializer(app) - self.user_deserializer = users.UserDeserializer(app) - self.api_key_manager = api_keys.ApiKeyManager(app) +class UserAPIController(BaseGalaxyAPIController, UsesTagsMixin, BaseUIController, UsesFormDefinitionsMixin): + user_manager: users.UserManager = depends(users.UserManager) + user_serializer: users.UserSerializer = depends(users.UserSerializer) + user_deserializer: users.UserDeserializer = depends(users.UserDeserializer) + api_key_manager: api_keys.ApiKeyManager = depends(api_keys.ApiKeyManager) @expose_api def index(self, trans: ProvidesUserContext, deleted='False', f_email=None, f_name=None, f_any=None, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/visualizations.py b/lib/galaxy/webapps/galaxy/api/visualizations.py index dc14e6fa6d1..de633671b20 100644 --- a/lib/galaxy/webapps/galaxy/api/visualizations.py +++ b/lib/galaxy/webapps/galaxy/api/visualizations.py @@ -20,24 +20,21 @@ from galaxy.managers.visualizations import ( from galaxy.model.item_attrs import UsesAnnotations from galaxy.web import expose_api from galaxy.webapps.base.controller import ( - BaseAPIController, SharableMixin, UsesVisualizationMixin ) from galaxy.webapps.base.webapp import GalaxyWebTransaction +from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) -class VisualizationsController(BaseAPIController, UsesVisualizationMixin, SharableMixin, UsesAnnotations): +class VisualizationsController(BaseGalaxyAPIController, UsesVisualizationMixin, SharableMixin, UsesAnnotations): """ RESTful controller for interactions with visualizations. """ - - def __init__(self, app): - super().__init__(app) - self.manager = VisualizationManager(app) - self.serializer = VisualizationSerializer(app) + manager: VisualizationManager = depends(VisualizationManager) + serializer: VisualizationSerializer = depends(VisualizationSerializer) @expose_api def index(self, trans: GalaxyWebTransaction, **kwargs): diff --git a/lib/galaxy/webapps/galaxy/api/webhooks.py b/lib/galaxy/webapps/galaxy/api/webhooks.py index cec2a6201e8..1761777500d 100644 --- a/lib/galaxy/webapps/galaxy/api/webhooks.py +++ b/lib/galaxy/webapps/galaxy/api/webhooks.py @@ -6,13 +6,13 @@ import logging from typing import Any from galaxy.web import expose_api_anonymous_and_sessionless -from galaxy.webapps.base.controller import BaseAPIController from galaxy.webapps.base.webapp import GalaxyWebTransaction +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class WebhooksController(BaseAPIController): +class WebhooksController(BaseGalaxyAPIController): @expose_api_anonymous_and_sessionless def all_webhooks(self, trans: GalaxyWebTransaction, **kwd): diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 02ffda95cca..5e5380332b4 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -28,6 +28,7 @@ from galaxy.managers.workflows import ( WorkflowUpdateOptions, ) from galaxy.model.item_attrs import UsesAnnotations +from galaxy.structured_app import StructuredApp from galaxy.tool_shed.galaxy_install.install_manager import InstallRepositoryManager from galaxy.tools import recommendations from galaxy.tools.parameters import populate_state @@ -42,7 +43,6 @@ from galaxy.web import ( format_return_as_json, ) from galaxy.webapps.base.controller import ( - BaseAPIController, SharableMixin, url_for, UsesStoredWorkflowMixin @@ -52,13 +52,14 @@ from galaxy.workflow.extract import extract_workflow from galaxy.workflow.modules import module_factory from galaxy.workflow.run import invoke, queue_invoke from galaxy.workflow.run_request import build_workflow_run_configs +from . import BaseGalaxyAPIController log = logging.getLogger(__name__) -class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnnotations, SharableMixin): +class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, UsesAnnotations, SharableMixin): - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) self.history_manager = app.history_manager self.workflow_manager = app.workflow_manager diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 2c3453eca69..1af2bd8ea7f 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -29,7 +29,7 @@ log = logging.getLogger(__name__) class GalaxyWebApplication(galaxy.webapps.base.webapp.WebApplication): - pass + injection_aware = True def app_factory(*args, **kwargs): diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 78fd4fb931d..39d1c39da65 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -10,7 +10,6 @@ from markupsafe import escape from galaxy import ( datatypes, - managers, model, util, web @@ -18,7 +17,10 @@ from galaxy import ( from galaxy.datatypes import sniff from galaxy.datatypes.display_applications.util import decode_dataset_user, encode_dataset_user from galaxy.exceptions import RequestParameterInvalidException +from galaxy.managers.hdas import HDADeserializer, HDAManager +from galaxy.managers.histories import HistoryManager from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings +from galaxy.structured_app import StructuredApp from galaxy.util import ( inflector, sanitize_text, @@ -29,6 +31,7 @@ from galaxy.util.sanitize_html import sanitize_html from galaxy.web import form_builder from galaxy.web.framework.helpers import iff from galaxy.webapps.base.controller import BaseUIController, ERROR, SUCCESS, url_for, UsesExtendedMetadataMixin +from ..api import depends log = logging.getLogger(__name__) @@ -42,12 +45,12 @@ except ImportError: class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesExtendedMetadataMixin): + history_manager: HistoryManager = depends(HistoryManager) + hda_manager: HDAManager = depends(HDAManager) + hda_deserializer: HDADeserializer = depends(HDADeserializer) - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) - self.history_manager = managers.histories.HistoryManager(app) - self.hda_manager = managers.hdas.HDAManager(app) - self.hda_deserializer = managers.hdas.HDADeserializer(app) def _get_job_for_dataset(self, trans, dataset_id): ''' diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index 996f312696a..24734b0047d 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -15,13 +15,14 @@ from sqlalchemy.orm import ( import galaxy.util from galaxy import exceptions -from galaxy import managers from galaxy import model from galaxy import web +from galaxy.managers import histories from galaxy.model.item_attrs import ( UsesAnnotations, UsesItemRatings ) +from galaxy.structured_app import StructuredApp from galaxy.util import ( listify, Params, @@ -47,6 +48,7 @@ from galaxy.webapps.base.controller import ( WARNING, ) from ._create_history_template import render_item +from ..api import depends log = logging.getLogger(__name__) @@ -235,12 +237,12 @@ class HistoryAllPublishedGrid(grids.Grid): class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesItemRatings, ExportsHistoryMixin, ImportsHistoryMixin): + history_manager: histories.HistoryManager = depends(histories.HistoryManager) + history_export_view: histories.HistoryExportView = depends(histories.HistoryExportView) + history_serializer: histories.HistorySerializer = depends(histories.HistorySerializer) - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) - self.history_manager = managers.histories.HistoryManager(app) - self.history_export_view = managers.histories.HistoryExportView(app) - self.history_serializer = managers.histories.HistorySerializer(self.app) @web.expose def index(self, trans): diff --git a/lib/galaxy/webapps/galaxy/controllers/openid.py b/lib/galaxy/webapps/galaxy/controllers/openid.py index a117676afc3..d7872f275f4 100644 --- a/lib/galaxy/webapps/galaxy/controllers/openid.py +++ b/lib/galaxy/webapps/galaxy/controllers/openid.py @@ -7,6 +7,7 @@ import logging from galaxy import web from galaxy.openid.openid_manager import OpenIDManager from galaxy.openid.providers import OpenIDProviders +from galaxy.structured_app import StructuredApp from galaxy.util import unicodify from galaxy.web import url_for from galaxy.webapps.base.controller import BaseUIController @@ -16,7 +17,7 @@ log = logging.getLogger(__name__) class OpenID(BaseUIController): - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) if app.config.enable_openid: self.openid_manager = OpenIDManager(app.config.openid_consumer_cache_path) diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index 2a5c70117a6..25c79ae4477 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -25,6 +25,7 @@ from galaxy.managers.pages import ( ) from galaxy.managers.workflows import WorkflowsManager from galaxy.model.item_attrs import UsesItemRatings +from galaxy.structured_app import StructuredApp from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html from galaxy.web import ( @@ -41,6 +42,7 @@ from galaxy.webapps.base.controller import ( UsesStoredWorkflowMixin, UsesVisualizationMixin ) +from ..api import depends def format_bool(b): @@ -269,14 +271,14 @@ class PageController(BaseUIController, SharableMixin, _datasets_selection_grid = HistoryDatasetAssociationSelectionGrid() _page_selection_grid = PageSelectionGrid() _visualization_selection_grid = VisualizationSelectionGrid() + page_manager: PageManager = depends(PageManager) + history_manager: HistoryManager = depends(HistoryManager) + history_serializer: HistorySerializer = depends(HistorySerializer) + hda_manager: HDAManager = depends(HDAManager) + workflow_manager: WorkflowsManager = depends(WorkflowsManager) - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) - self.page_manager = PageManager(app) - self.history_manager = HistoryManager(app) - self.history_serializer = HistorySerializer(self.app) - self.hda_manager = HDAManager(app) - self.workflow_manager = WorkflowsManager(app) @web.expose @web.json diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 3109a0519cb..1d7feaf2c7c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -13,10 +13,11 @@ from webob.exc import ( ) from galaxy import ( - managers, web ) +from galaxy.managers.histories import HistoryManager, HistorySerializer from galaxy.model.item_attrs import UsesAnnotations +from galaxy.structured_app import StructuredApp from galaxy.util import ( FILENAME_VALID_CHARS, listify, @@ -24,6 +25,7 @@ from galaxy.util import ( unicodify, ) from galaxy.webapps.base import controller +from ..api import depends log = logging.getLogger(__name__) @@ -33,11 +35,11 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): """ Controller class that maps to the url root of Galaxy (i.e. '/'). """ + history_manager: HistoryManager = depends(HistoryManager) + history_serializer: HistorySerializer = depends(HistorySerializer) - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) - self.history_manager = managers.histories.HistoryManager(app) - self.history_serializer = managers.histories.HistorySerializer(app) @web.expose def default(self, trans, target1=None, target2=None, **kwd): diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py index 00cd0c400cb..1df089bd00e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -20,6 +20,7 @@ from galaxy.security.validate_user_input import ( validate_email, validate_publicname ) +from galaxy.structured_app import StructuredApp from galaxy.web import expose_api_anonymous_and_sessionless from galaxy.web import url_for from galaxy.webapps.base.controller import ( @@ -27,6 +28,7 @@ from galaxy.webapps.base.controller import ( CreatesApiKeysMixin, UsesFormDefinitionsMixin ) +from ..api import depends log = logging.getLogger(__name__) @@ -36,11 +38,11 @@ def _filtered_registration_params_dict(payload): class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin): + user_manager: users.UserManager = depends(users.UserManager) installed_len_files = None - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) - self.user_manager = users.UserManager(app) def __handle_role_and_group_auto_creation(self, trans, user, roles, auto_create_roles=False, auto_create_groups=False, auto_assign_roles_to_groups_only=False): diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index ea8e4857a69..74dfa7f94e9 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -18,9 +18,11 @@ from sqlalchemy import ( ) from sqlalchemy.orm import eagerload, undefer -from galaxy import managers, model, util, web +from galaxy import model, util, web from galaxy.datatypes.interval import Bed +from galaxy.managers.hdas import HDAManager from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings +from galaxy.structured_app import StructuredApp from galaxy.util import sanitize_text, unicodify from galaxy.util.sanitize_html import sanitize_html from galaxy.visualization.data_providers.genome import RawBedDataProvider @@ -34,6 +36,7 @@ from galaxy.webapps.base.controller import ( SharableMixin, UsesVisualizationMixin ) +from ..api import depends log = logging.getLogger(__name__) @@ -231,10 +234,10 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization _history_datasets_grid = HistoryDatasetsSelectionGrid() _library_datasets_grid = LibraryDatasetsSelectionGrid() _tracks_grid = TracksterSelectionGrid() + hda_manager: HDAManager = depends(HDAManager) - def __init__(self, app): + def __init__(self, app: StructuredApp): super().__init__(app) - self.hda_manager = managers.hdas.HDAManager(app) # # -- Functions for listing visualizations. -- diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index c6fd1b8d2bf..954d9c45b1c 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -71,11 +71,11 @@ def add_exception_handler( ) -> None: @app.exception_handler(RequestValidationError) - async def validate_exception_middleware(request: Request, exc: MessageException) -> Response: + async def validate_exception_middleware(request: Request, exc: RequestValidationError) -> Response: exc = validation_error_to_message_exception(exc) error_dict = api_error_message(None, exception=exc) return JSONResponse( - status_code=exc.status_code, + status_code=400, content=error_dict ) diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py index c08167b5ce4..1feeef2df60 100644 --- a/lib/galaxy/workflow/run_request.py +++ b/lib/galaxy/workflow/run_request.py @@ -6,7 +6,6 @@ from galaxy import ( exceptions, model ) -from galaxy.managers import histories from galaxy.tools.parameters.meta import expand_workflow_inputs from galaxy.workflow.resources import get_resource_mapper_function @@ -202,7 +201,7 @@ def _get_target_history(trans, workflow, payload, param_keys=None, index=0): else: history_name = history_param if history_id: - history_manager = histories.HistoryManager(trans.app) + history_manager = trans.app.history_manager target_history = history_manager.get_owned(trans.security.decode_id(history_id), trans.user, current_history=trans.history) else: if history_name: diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index 94758e85e2c..3457f9e9f13 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -1,16 +1,24 @@ import logging import sys import time +from typing import Any + +from sqlalchemy.orm.scoping import scoped_session import galaxy.datatypes.registry import galaxy.tools.data import tool_shed.repository_registry import tool_shed.repository_types.registry import tool_shed.webapp.model +from galaxy import auth from galaxy.config import configure_logging +from galaxy.managers.citations import CitationsManager +from galaxy.managers.users import UserManager +from galaxy.model.base import SharedModelMapping from galaxy.model.tags import CommunityTagHandler -from galaxy.quota import NoQuotaAgent +from galaxy.quota import NoQuotaAgent, QuotaAgent from galaxy.security import idencoding +from galaxy.structured_app import BasicApp from galaxy.util.dbkeys import GenomeBuilds from galaxy.web_stack import application_stack_instance from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager @@ -20,16 +28,18 @@ from . import config log = logging.getLogger(__name__) -class UniverseApplication: +class UniverseApplication(BasicApp): """Encapsulates the state of a Universe application""" - def __init__(self, **kwd): + def __init__(self, **kwd) -> None: + super().__init__() + self[BasicApp] = self log.debug("python path is: %s", ", ".join(sys.path)) self.name = "tool_shed" # will be overwritten when building WSGI app self.is_webapp = False # Read the tool_shed.ini configuration file and check for errors. - self.config = config.Configuration(**kwd) + self.config: Any = config.Configuration(**kwd) self.config.check() configure_logging(self.config) self.application_stack = application_stack_instance() @@ -50,28 +60,34 @@ class UniverseApplication: create_or_verify_database(db_url, self.config.database_engine_options) # Set up the Tool Shed database engine and ORM. from tool_shed.webapp.model import mapping - self.model = mapping.init(self.config.file_path, - db_url, - self.config.database_engine_options) + model: mapping.ToolShedModelMapping = mapping.init( + self.config.file_path, + db_url, + self.config.database_engine_options + ) + self.model = model self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret) + self._register_singleton(idencoding.IdEncodingHelper, self.security) + self._register_singleton(SharedModelMapping, model) + self._register_singleton(mapping.ToolShedModelMapping, model) + self._register_singleton(scoped_session, self.model.context) + self._register_singleton(UserManager, UserManager) # initialize the Tool Shed tag handler. self.tag_handler = CommunityTagHandler(self) # Initialize the Tool Shed tool data tables. Never pass a configuration file here # because the Tool Shed should always have an empty dictionary! self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(self.config.tool_data_path) self.genome_builds = GenomeBuilds(self) - from galaxy import auth - self.auth_manager = auth.AuthManager(self) + self.auth_manager = self._register_singleton(auth.AuthManager, auth.AuthManager(self.config)) # Citation manager needed to load tools. - from galaxy.managers.citations import CitationsManager - self.citations_manager = CitationsManager(self) + self.citations_manager = self._register_singleton(CitationsManager, CitationsManager(self)) self.use_tool_dependency_resolution = False # Initialize the Tool Shed security agent. - self.security_agent = self.model.security_agent + self.security_agent = model.security_agent # The Tool Shed makes no use of a quota, but this attribute is still required. - self.quota_agent = NoQuotaAgent() + self.quota_agent = self._register_singleton(QuotaAgent, NoQuotaAgent()) # Initialize the baseline Tool Shed statistics component. - self.shed_counter = self.model.shed_counter + self.shed_counter = model.shed_counter # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir diff --git a/lib/tool_shed/webapp/model/mapping.py b/lib/tool_shed/webapp/model/mapping.py index 271aded24f1..e0a2f874c40 100644 --- a/lib/tool_shed/webapp/model/mapping.py +++ b/lib/tool_shed/webapp/model/mapping.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import backref, mapper, relation import tool_shed.webapp.model import tool_shed.webapp.util.shed_statistics as shed_statistics -from galaxy.model.base import ModelMapping +from galaxy.model.base import SharedModelMapping from galaxy.model.custom_types import JSONType, TrimmedString from galaxy.model.orm.engine_factory import build_engine from galaxy.model.orm.now import now @@ -310,7 +310,13 @@ mapper(RepositoryCategoryAssociation, RepositoryCategoryAssociation.table, repository=relation(Repository))) -def init(file_path, url, engine_options=None, create_tables=False): +class ToolShedModelMapping(SharedModelMapping): + security_agent: CommunityRBACAgent + shed_counter: shed_statistics.ShedCounter + create_tables: bool + + +def init(file_path, url, engine_options=None, create_tables=False) -> ToolShedModelMapping: """Connect mappings to the database""" engine_options = engine_options or {} # Create the database engine @@ -318,7 +324,7 @@ def init(file_path, url, engine_options=None, create_tables=False): # Connect the metadata to the database. metadata.bind = engine - result = ModelMapping([tool_shed.webapp.model], engine=engine) + result = ToolShedModelMapping([tool_shed.webapp.model], engine=engine) if create_tables: metadata.create_all() diff --git a/packages/app/galaxy/di.py b/packages/app/galaxy/di.py new file mode 120000 index 00000000000..0efc76dcc24 --- /dev/null +++ b/packages/app/galaxy/di.py @@ -0,0 +1 @@ +../../../lib/galaxy/di.py \ No newline at end of file diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt index 0912b8ae2d1..ac105577ec7 100644 --- a/packages/app/requirements.txt +++ b/packages/app/requirements.txt @@ -17,3 +17,4 @@ sortedcontainers Markdown refgenconf Fabric3 +lagom diff --git a/packages/webapps/requirements.txt b/packages/webapps/requirements.txt index 8bb9841f7b5..d4641af2264 100644 --- a/packages/webapps/requirements.txt +++ b/packages/webapps/requirements.txt @@ -1,3 +1,5 @@ galaxy-app Cheetah3 Mako +fastapi +starlette \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 31b78e5261f..00608e54a9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ gxformat2 = "*" h5py = "!=2.7.0, !=2.7.1" isa-rwval = "*" kombu = "*" +lagom = "*" Mako = "*" Markdown = "*" MarkupSafe = "*" @@ -70,6 +71,7 @@ sqlalchemy-migrate = "*" SQLAlchemy-Utils = "!=0.36.8" # https://github.com/kvesteri/sqlalchemy-utils/issues/462 sqlitedict = "*" sqlparse = "*" +starlette = "*" starlette-context = "*" svgwrite = "*" uvicorn = "*" diff --git a/test/unit/managers/base.py b/test/unit/managers/base.py index 9c8469cdaf7..0dfec760cff 100644 --- a/test/unit/managers/base.py +++ b/test/unit/managers/base.py @@ -46,7 +46,7 @@ class BaseTestCase(unittest.TestCase): self.trans.app.config.is_admin_user = mock_is_admin_user def set_up_managers(self): - self.user_manager = UserManager(self.app) + self.user_manager = self.app[UserManager] def set_up_trans(self): self.admin_user = self.user_manager.create(email=admin_email, username='admin', password=default_password) diff --git a/test/unit/managers/test_CollectionManager.py b/test/unit/managers/test_CollectionManager.py index 5eaf13d0a83..410ea1c99cb 100644 --- a/test/unit/managers/test_CollectionManager.py +++ b/test/unit/managers/test_CollectionManager.py @@ -21,10 +21,10 @@ class DatasetCollectionManagerTestCase(BaseTestCase, CreatesCollectionsMixin): def set_up_managers(self): super().set_up_managers() - self.dataset_manager = DatasetManager(self.app) - self.hda_manager = HDAManager(self.app) - self.history_manager = HistoryManager(self.app) - self.collection_manager = DatasetCollectionManager(self.app) + self.dataset_manager = self.app[DatasetManager] + self.hda_manager = self.app[HDAManager] + self.history_manager = self.app[HistoryManager] + self.collection_manager = self.app[DatasetCollectionManager] def test_create_simple_list(self): owner = self.user_manager.create(**user2_data) diff --git a/test/unit/managers/test_DatasetManager.py b/test/unit/managers/test_DatasetManager.py index 3e00699f568..c070b3583f5 100644 --- a/test/unit/managers/test_DatasetManager.py +++ b/test/unit/managers/test_DatasetManager.py @@ -216,7 +216,7 @@ class DatasetSerializerTestCase(BaseTestCase): def set_up_managers(self): super().set_up_managers() self.dataset_manager = DatasetManager(self.app) - self.dataset_serializer = DatasetSerializer(self.app) + self.dataset_serializer = DatasetSerializer(self.app, self.user_manager) self.role_manager = RoleManager(self.app) def test_views(self): diff --git a/test/unit/managers/test_HDAManager.py b/test/unit/managers/test_HDAManager.py index a4068918957..8f44aaeb47b 100644 --- a/test/unit/managers/test_HDAManager.py +++ b/test/unit/managers/test_HDAManager.py @@ -19,9 +19,9 @@ class HDATestCase(BaseTestCase): def set_up_managers(self): super().set_up_managers() - self.hda_manager = hdas.HDAManager(self.app) - self.history_manager = HistoryManager(self.app) - self.dataset_manager = DatasetManager(self.app) + self.hda_manager = self.app[hdas.HDAManager] + self.history_manager = self.app[HistoryManager] + self.dataset_manager = self.app[DatasetManager] def _create_vanilla_hda(self, user_data=None): user_data = user_data or user2_data diff --git a/test/unit/managers/test_HDCAManager.py b/test/unit/managers/test_HDCAManager.py index 7ac099f0466..c9a3f250e8b 100644 --- a/test/unit/managers/test_HDCAManager.py +++ b/test/unit/managers/test_HDCAManager.py @@ -23,11 +23,11 @@ class HDCATestCase(BaseTestCase, CreatesCollectionsMixin): def set_up_managers(self): super().set_up_managers() - self.hdca_manager = hdcas.HDCAManager(self.app) - self.hda_manager = hdas.HDAManager(self.app) - self.history_manager = HistoryManager(self.app) - self.dataset_manager = DatasetManager(self.app) - self.collection_manager = collections.DatasetCollectionManager(self.app) + self.hdca_manager = self.app[hdcas.HDCAManager] + self.hda_manager = self.app[hdas.HDAManager] + self.history_manager = self.app[HistoryManager] + self.dataset_manager = self.app[DatasetManager] + self.collection_manager = self.app[collections.DatasetCollectionManager] def _create_history(self, user_data=None, **kwargs): user_data = user_data or user2_data diff --git a/test/unit/managers/test_HistoryContentsManager.py b/test/unit/managers/test_HistoryContentsManager.py index b601d4ead6d..eeb6c34d9fc 100644 --- a/test/unit/managers/test_HistoryContentsManager.py +++ b/test/unit/managers/test_HistoryContentsManager.py @@ -23,11 +23,11 @@ class HistoryAsContainerBaseTestCase(BaseTestCase, CreatesCollectionsMixin): def set_up_managers(self): super().set_up_managers() - self.history_manager = HistoryManager(self.app) - self.hda_manager = hdas.HDAManager(self.app) - self.collection_manager = collections.DatasetCollectionManager(self.app) - self.contents_manager = history_contents.HistoryContentsManager(self.app) - self.history_contents_filters = history_contents.HistoryContentsFilters(self.app) + self.history_manager = self.app[HistoryManager] + self.hda_manager = self.app[hdas.HDAManager] + self.collection_manager = self.app[collections.DatasetCollectionManager] + self.contents_manager = self.app[history_contents.HistoryContentsManager] + self.history_contents_filters = self.app[history_contents.HistoryContentsFilters] def add_hda_to_history(self, history, **kwargs): dataset = self.hda_manager.dataset_manager.create() diff --git a/test/unit/managers/test_HistoryManager.py b/test/unit/managers/test_HistoryManager.py index d4c22d25bbf..d2c38e2de88 100644 --- a/test/unit/managers/test_HistoryManager.py +++ b/test/unit/managers/test_HistoryManager.py @@ -34,8 +34,8 @@ class HistoryManagerTestCase(BaseTestCase): def set_up_managers(self): super().set_up_managers() - self.history_manager = HistoryManager(self.app) - self.hda_manager = hdas.HDAManager(self.app) + self.history_manager = self.app[HistoryManager] + self.hda_manager = self.app[hdas.HDAManager] def add_hda_to_history(self, history, **kwargs): dataset = self.hda_manager.dataset_manager.create() @@ -399,9 +399,9 @@ class HistorySerializerTestCase(BaseTestCase): def set_up_managers(self): super().set_up_managers() - self.history_manager = HistoryManager(self.app) - self.hda_manager = hdas.HDAManager(self.app) - self.history_serializer = HistorySerializer(self.app) + self.history_manager = self.app[HistoryManager] + self.hda_manager = self.app[hdas.HDAManager] + self.history_serializer = self.app[HistorySerializer] def test_views(self): user2 = self.user_manager.create(**user2_data) @@ -620,8 +620,8 @@ class HistoryDeserializerTestCase(BaseTestCase): def set_up_managers(self): super().set_up_managers() - self.history_manager = HistoryManager(self.app) - self.history_deserializer = HistoryDeserializer(self.app) + self.history_manager = self.app[HistoryManager] + self.history_deserializer = self.app[HistoryDeserializer] def test_ratings(self): user2 = self.user_manager.create(**user2_data) @@ -682,8 +682,8 @@ class HistoryFiltersTestCase(BaseTestCase): def set_up_managers(self): super().set_up_managers() - self.history_manager = HistoryManager(self.app) - self.filter_parser = HistoryFilters(self.app) + self.history_manager = self.app[HistoryManager] + self.filter_parser = self.app[HistoryFilters] # ---- functional and orm filter splitting and resolution def test_parse_filters(self): diff --git a/test/unit/managers/test_TagHandler.py b/test/unit/managers/test_TagHandler.py index 76f9c288ef4..c2982662203 100644 --- a/test/unit/managers/test_TagHandler.py +++ b/test/unit/managers/test_TagHandler.py @@ -15,17 +15,17 @@ class TagHandlerTestCase(BaseTestCase): def set_up_managers(self): super().set_up_managers() - self.hda_manager = hdas.HDAManager(self.app) - self.history_manager = HistoryManager(self.app) - self.dataset_manager = DatasetManager(self.app) - self.tag_handler = GalaxyTagHandler(self.trans.sa_session) + self.app.hda_manager = self.app[hdas.HDAManager] + self.app.history_manager = self.app[HistoryManager] + self.app.dataset_manager = self.app[DatasetManager] + self.tag_handler = self.app[GalaxyTagHandler] self.user = self.user_manager.create(**user2_data) def _create_vanilla_hda(self, user=None): owner = user or self.user - history1 = self.history_manager.create(name='history1', user=owner) - dataset1 = self.dataset_manager.create() - return self.hda_manager.create(history=history1, dataset=dataset1) + history1 = self.app.history_manager.create(name='history1', user=owner) + dataset1 = self.app.dataset_manager.create() + return self.app.hda_manager.create(history=history1, dataset=dataset1) def _check_tag_list(self, tags, expected_tags): self.assertEqual(len(tags), len(expected_tags)) diff --git a/test/unit/managers/test_UserManager.py b/test/unit/managers/test_UserManager.py index bf2f48a7e76..df08bcfc3dc 100644 --- a/test/unit/managers/test_UserManager.py +++ b/test/unit/managers/test_UserManager.py @@ -75,14 +75,10 @@ class UserManagerTestCase(BaseTestCase): def test_email_queries(self): user2 = self.user_manager.create(**user2_data) - user3 = self.user_manager.create(**user3_data) self.log("should be able to query by email") self.assertEqual(self.user_manager.by_email(user2_data['email']), user2) - # note: sorted by email alpha - self.assertEqual(self.user_manager.by_email_like('%@%'), [self.admin_user, user2, user3]) - def test_admin(self): user2 = self.user_manager.create(**user2_data) @@ -108,21 +104,6 @@ class UserManagerTestCase(BaseTestCase): self.assertEqual(self.user_manager.current_user(self.trans), self.admin_user) self.assertNotEqual(self.user_manager.current_user(self.trans), user2) - def test_api_keys(self): - user2 = self.user_manager.create(**user2_data) - - self.log("should return None if no APIKey has been created") - self.assertEqual(self.user_manager.valid_api_key(user2), None) - - self.log("should be able to generate and retrieve valid api key") - user2_api_key = self.user_manager.create_api_key(user2) - self.assertIsInstance(user2_api_key, str) - self.assertEqual(self.user_manager.valid_api_key(user2).key, user2_api_key) - - self.log("should return the most recent (i.e. most valid) api key") - user2_api_key_2 = self.user_manager.create_api_key(user2) - self.assertEqual(self.user_manager.valid_api_key(user2).key, user2_api_key_2) - def test_change_password(self): self.log("should be able to change password") user2 = self.user_manager.create(**user2_data) @@ -289,7 +270,7 @@ class CurrentUserSerializerTestCase(BaseTestCase): def set_up_managers(self): super().set_up_managers() - self.history_manager = histories.HistoryManager(self.app) + self.history_manager = self.app[histories.HistoryManager] self.user_serializer = users.CurrentUserSerializer(self.app) def test_anonymous(self): diff --git a/test/unit/tools_support.py b/test/unit/tools_support.py index 82e0f73e2b3..5c86cd64dbd 100644 --- a/test/unit/tools_support.py +++ b/test/unit/tools_support.py @@ -29,6 +29,7 @@ class UsesApp: self.app = galaxy_mock.MockApp() self.app.config.new_file_path = os.path.join(self.test_directory, "new_files") self.app.config.admin_users = "mary@example.com" + self.app.job_search = None def tear_down_app(self): shutil.rmtree(self.test_directory) diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index fe16971e2e4..914077f063d 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -5,17 +5,23 @@ import os import shutil import tempfile +from sqlalchemy.orm.scoping import scoped_session + from galaxy import ( + di, model, objectstore, - quota + quota, ) from galaxy.auth import AuthManager from galaxy.datatypes import registry from galaxy.jobs.manager import NoopManager from galaxy.managers.users import UserManager from galaxy.model import mapping, tags +from galaxy.model.base import SharedModelMapping +from galaxy.model.mapping import GalaxyModelMapping from galaxy.security import idencoding +from galaxy.structured_app import BasicApp, StructuredApp from galaxy.tool_util.deps.containers import NullContainerFinder from galaxy.util import StructuredExecutionTimer from galaxy.util.bunch import Bunch @@ -56,17 +62,25 @@ def buildMockEnviron(**kwargs): return environ -class MockApp: +class MockApp(di.Container): def __init__(self, config=None, **kwargs): + super().__init__() + self[BasicApp] = self + self[StructuredApp] = self self.config = config or MockAppConfig(**kwargs) self.security = self.config.security + self[idencoding.IdEncodingHelper] = self.security self.name = kwargs.get('name', 'galaxy') self.object_store = objectstore.build_object_store_from_config(self.config) self.model = mapping.init("/tmp", self.config.database_connection, create_tables=True, object_store=self.object_store) + self[SharedModelMapping] = self.model + self[GalaxyModelMapping] = self.model + self[scoped_session] = self.model.context self.security_agent = self.model.security_agent self.visualizations_registry = MockVisualizationsRegistry() self.tag_handler = tags.GalaxyTagHandler(self.model.context) + self[tags.GalaxyTagHandler] = self.tag_handler self.quota_agent = quota.DatabaseQuotaAgent(self.model) self.init_datatypes() self.job_config = Bunch( @@ -81,7 +95,7 @@ class MockApp: self.genome_builds = GenomeBuilds(self) self.job_manager = NoopManager() self.application_stack = ApplicationStack() - self.auth_manager = AuthManager(self) + self.auth_manager = AuthManager(self.config) self.user_manager = UserManager(self) self.execution_timer_factory = Bunch(get_timer=StructuredExecutionTimer) From 377f836352f47047b692e7bfc024cc7b88a5095b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 23 Feb 2021 17:09:11 +0100 Subject: [PATCH 54/70] Set default USE_UVICORN to true only on python >= 3.7 Fixes test exception in https://github.com/galaxyproject/planemo/pull/1142#issuecomment-784308435 --- lib/galaxy_test/driver/driver_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index f1b52d91be8..8f8681fb8ce 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -78,7 +78,8 @@ uwsgi: """) DEFAULT_LOCALES = "en" -USE_UVICORN = asbool(os.environ.get('GALAXY_TEST_USE_UVICORN', True)) +CAN_BUILD_ASGI_APP = sys.version_info[:2] >= (3, 7) +USE_UVICORN = asbool(os.environ.get('GALAXY_TEST_USE_UVICORN', CAN_BUILD_ASGI_APP)) log = logging.getLogger("test_driver") From b574009c96c80c1014e6749732bc096ab2de2973 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 23 Feb 2021 22:29:32 +0100 Subject: [PATCH 55/70] small changes to upgrade_all_steps refactor action and add a test --- .../src/components/Workflow/Editor/Index.vue | 2 +- lib/galaxy/workflow/refactor/execute.py | 12 +++--- lib/galaxy/workflow/refactor/schema.py | 6 +-- lib/galaxy_test/base/workflow_fixtures.py | 39 +++++++++++++++++++ test/integration/test_workflow_refactoring.py | 35 +++++++++++++++++ 5 files changed, 84 insertions(+), 10 deletions(-) diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue index 8162287648a..cb336c053a6 100644 --- a/client/src/components/Workflow/Editor/Index.vue +++ b/client/src/components/Workflow/Editor/Index.vue @@ -424,7 +424,7 @@ export default { showLint(); }, onUpgrade() { - this.attemptRefactor([{'action_type': 'upgrade_workflow'}]); + this.attemptRefactor([{ action_type: "upgrade_all_steps" }]); }, onEdit() { this.isCanvas = true; diff --git a/lib/galaxy/workflow/refactor/execute.py b/lib/galaxy/workflow/refactor/execute.py index ba2ee705ebe..7053e4a3508 100644 --- a/lib/galaxy/workflow/refactor/execute.py +++ b/lib/galaxy/workflow/refactor/execute.py @@ -38,9 +38,9 @@ from .schema import ( UpdateReportAction, UpdateStepLabelAction, UpdateStepPositionAction, + UpgradeAllStepsAction, UpgradeSubworkflowAction, UpgradeToolAction, - UpgradeWorkflowAction, ) from ..modules import ( InputParameterModule, @@ -393,14 +393,14 @@ class WorkflowRefactorExecutor: step_def["tool_state"] = step.module.get_tool_state() self._patch_step(execution, step, step_def) - def _apply_upgrade_workflow(self, action: UpgradeWorkflowAction, execution: RefactorActionExecution): + def _apply_upgrade_all_steps(self, action: UpgradeAllStepsAction, execution: RefactorActionExecution): for step_order_index, step in self._as_dict["steps"].items(): if step.get('type') == 'subworkflow': - action = UpgradeSubworkflowAction(action_type='upgrade_subworkflow', step={'order_index': step_order_index}) - self._apply_upgrade_subworkflow(action, execution) + step_action = UpgradeSubworkflowAction(action_type='upgrade_subworkflow', step={'order_index': step_order_index}) + self._apply_upgrade_subworkflow(step_action, execution) elif step.get('type') == 'tool': - action = UpgradeToolAction(action_type='upgrade_tool', step={'order_index': step_order_index}) - self._apply_upgrade_tool(action, execution) + step_action = UpgradeToolAction(action_type='upgrade_tool', step={'order_index': step_order_index}) # type: ignore + self._apply_upgrade_tool(step_action, execution) # type: ignore def _find_step(self, step_reference: step_reference_union): order_index = None diff --git a/lib/galaxy/workflow/refactor/schema.py b/lib/galaxy/workflow/refactor/schema.py index 37eeda5c6c0..e19c13dfdc0 100644 --- a/lib/galaxy/workflow/refactor/schema.py +++ b/lib/galaxy/workflow/refactor/schema.py @@ -211,8 +211,8 @@ class UpgradeToolAction(BaseAction): tool_version: Optional[str] -class UpgradeWorkflowAction(BaseAction): - action_type: Literal['upgrade_workflow'] +class UpgradeAllStepsAction(BaseAction): + action_type: Literal['upgrade_all_steps'] union_action_classes = Union[ @@ -234,7 +234,7 @@ union_action_classes = Union[ UpdateStepPositionAction, UpgradeSubworkflowAction, UpgradeToolAction, - UpgradeWorkflowAction, + UpgradeAllStepsAction, RemoveUnlabeledWorkflowOutputs, ] diff --git a/lib/galaxy_test/base/workflow_fixtures.py b/lib/galaxy_test/base/workflow_fixtures.py index 9f9e006799f..704d51f258c 100644 --- a/lib/galaxy_test/base/workflow_fixtures.py +++ b/lib/galaxy_test/base/workflow_fixtures.py @@ -375,6 +375,45 @@ steps: inner_input: outer_input """ +WORKFLOW_NESTED_WITH_MULTIPLE_VERSIONS_TOOL = """ +class: GalaxyWorkflow +inputs: + outer_input: data +outputs: + outer_output: + outputSource: cat/out_file1 +steps: + tool_update_step: + tool_id: multiple_versions + tool_version: '0.1' + in: + input1: outer_input + nested_workflow: + run: + class: GalaxyWorkflow + inputs: + inner_input: data + outputs: + workflow_output: + outputSource: random_lines/out_file1 + steps: + random_lines: + tool_id: random_lines1 + state: + num_lines: 1 + input: + $link: inner_input + seed_source: + seed_source_selector: set_seed + seed: asdf + in: + inner_input: tool_update_step/out_file1 + cat: + tool_id: cat1 + in: + input1: nested_workflow/workflow_output + queries_0|input2: nested_workflow/workflow_output +""" WORKFLOW_WITH_OUTPUT_ACTIONS = """ class: GalaxyWorkflow diff --git a/test/integration/test_workflow_refactoring.py b/test/integration/test_workflow_refactoring.py index d842e738b94..ec6dedcb1fc 100644 --- a/test/integration/test_workflow_refactoring.py +++ b/test/integration/test_workflow_refactoring.py @@ -22,6 +22,7 @@ from galaxy_test.base.populators import ( from galaxy_test.base.workflow_fixtures import ( WORKFLOW_NESTED_RUNTIME_PARAMETER, WORKFLOW_NESTED_SIMPLE, + WORKFLOW_NESTED_WITH_MULTIPLE_VERSIONS_TOOL, ) from galaxy_test.driver import integration_util @@ -691,6 +692,40 @@ steps: assert message.output_name == "workflow_output" assert message.output_label == "outer_output" + def test_upgrade_all_steps(self): + self.workflow_populator.upload_yaml_workflow(WORKFLOW_NESTED_WITH_MULTIPLE_VERSIONS_TOOL) + nested_stored_workflow = self._recent_stored_workflow(2) + assert self._latest_workflow.step_by_label("tool_update_step").tool_version == "0.1" + updated_nested_step = nested_stored_workflow.latest_workflow.step_by_label("random_lines") + assert updated_nested_step.tool_inputs["num_lines"] == "1" + + self._increment_nested_workflow_version( + nested_stored_workflow, + num_lines_from="1", + num_lines_to="2" + ) + self._app.model.session.expunge(nested_stored_workflow) + # ensure subworkflow updated properly... + nested_stored_workflow = self._recent_stored_workflow(2) + assert len(nested_stored_workflow.workflows) == 2 + actions = [ + {"action_type": "upgrade_all_steps"}, + ] + action_executions = self._refactor(actions).action_executions + assert self._latest_workflow.step_by_label("tool_update_step").tool_version == "0.2" + nested_stored_workflow = self._recent_stored_workflow(2) + updated_nested_step = nested_stored_workflow.latest_workflow.step_by_label("random_lines") + assert updated_nested_step.tool_inputs["num_lines"] == "2" + + assert len(action_executions) == 1 + messages = action_executions[0].messages + assert len(messages) == 1 + message = messages[0] + assert message.message_type == RefactorActionExecutionMessageTypeEnum.connection_drop_forced + assert message.order_index == 1 + assert message.step_label == "tool_update_step" + assert message.output_name == "output" + def _download_native(self, workflow=None): workflow = workflow or self._most_recent_stored_workflow workflow_id = self._app.security.encode_id(workflow.id) From f5929f8e060e749f736f9e9eb9ff3fb5da7f7c35 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 23 Feb 2021 19:20:11 -0500 Subject: [PATCH 56/70] Allow anonymous access to S3 buckets via file sources plugins. ```yaml - type: s3fs id: genomeark doc: Access to Genome Ark open data on AWS.. bucket: genomeark anon: true ``` 0;95;0c# Please enter the commit message for your changes. Lines starting --- lib/galaxy/dependencies/__init__.py | 8 ++++ .../dependencies/conditional-requirements.txt | 7 +++- test/unit/files/_util.py | 14 +++++++ test/unit/files/s3.py | 38 +++++++++++++++++++ test/unit/files/s3_file_sources_conf.yml | 5 +++ 5 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 test/unit/files/s3.py create mode 100644 test/unit/files/s3_file_sources_conf.yml diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index c15e872a993..6514fd77274 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -219,6 +219,14 @@ class ConditionalDependencies: def check_fs_webdavfs(self): return 'webdav' in self.file_sources + def check_fs_s3fs(self): + # pyfilesystem plugin access to s3 + return 's3' in self.file_sources + + def check_s3fs(self): + # use s3fs directly (skipping pyfilesystem) for direct access to more options + return 's3fs' in self.file_sources + def check_watchdog(self): install_set = {'auto', 'True', 'true', 'polling'} return (self.config['watch_tools'] in install_set diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index daeaa7d5d81..bffb7b3eb8b 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -13,8 +13,11 @@ python-ldap==3.2.0 python-pam galaxycloudrunner -fs.webdavfs -fs.dropboxfs +# For file sources plugins +fs.webdavfs # type: webdav +fs.dropboxfs # type: dropbox +fs-s3fs # type: s3 +s3fs # type: s3fs # Chronos client chronos-python==1.2.1 diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index fd5dfb41fe8..e85e9ba1743 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -74,6 +74,20 @@ def assert_realizes_as(file_sources, uri, expected, user_context=None): raise AssertionError(message) +def assert_realizes_contains(file_sources, uri, expected, user_context=None): + file_source_path = file_sources.get_file_source_path(uri) + with tempfile.NamedTemporaryFile(mode='r') as temp: + file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) + realized_contents = temp.read() + if expected not in realized_contents: + message = "Expected to realize contents at [{}] to contain [{}], instead found [{}]".format( + uri, + expected, + realized_contents, + ) + raise AssertionError(message) + + def write_from(file_sources, uri, content, user_context=None): file_source_path = file_sources.get_file_source_path(uri) with tempfile.NamedTemporaryFile(mode='w') as f: diff --git a/test/unit/files/s3.py b/test/unit/files/s3.py new file mode 100644 index 00000000000..cbae28b7948 --- /dev/null +++ b/test/unit/files/s3.py @@ -0,0 +1,38 @@ +import os + +import pytest + +from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig +from ._util import ( + assert_realizes_contains, + find, + user_context_fixture, +) +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "s3_file_sources_conf.yml") + +skip_if_not_slow = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_INCLUDE_SLOW'), + reason="GALAXY_TEST_INCLUDE_SLOW not set" +) + + +@skip_if_not_slow +def test_file_source(): + user_context = user_context_fixture() + file_sources = _configured_file_sources() + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + file_source = file_source_pair.file_source + res = file_source.list("/", recursive=False, user_context=user_context) + print(res) + dup = find(res, "File", "data_use_policies.txt") + assert dup + + assert_realizes_contains(file_sources, "gxfiles://test1/data_use_policies.txt", "DATA USE POLICIES", user_context=user_context) + + +def _configured_file_sources(conf_file=FILE_SOURCES_CONF): + file_sources_config = ConfiguredFileSourcesConfig() + return ConfiguredFileSources(file_sources_config, conf_file=conf_file) diff --git a/test/unit/files/s3_file_sources_conf.yml b/test/unit/files/s3_file_sources_conf.yml new file mode 100644 index 00000000000..087d6cf85d5 --- /dev/null +++ b/test/unit/files/s3_file_sources_conf.yml @@ -0,0 +1,5 @@ +- type: s3fs + id: test1 + doc: Test anonymous access to an AWS S3 bucket. + bucket: genomeark + anon: true From 935d429290c4971f1563e47dd3ecdc75e7df2658 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 24 Feb 2021 11:23:22 -0500 Subject: [PATCH 57/70] Missing file for s3 anonymous access fix PR. --- lib/galaxy/files/sources/s3fs.py | 86 ++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 lib/galaxy/files/sources/s3fs.py diff --git a/lib/galaxy/files/sources/s3fs.py b/lib/galaxy/files/sources/s3fs.py new file mode 100644 index 00000000000..b7a3fd63d3e --- /dev/null +++ b/lib/galaxy/files/sources/s3fs.py @@ -0,0 +1,86 @@ +import functools +import os + +try: + import s3fs +except ImportError: + s3fs = None + +from ..sources import BaseFilesSource + +DEFAULT_ENFORCE_SYMLINK_SECURITY = True +DEFAULT_DELETE_ON_REALIZE = False + + +class S3FsFilesSource(BaseFilesSource): + plugin_type = 's3fs' + + def __init__(self, **kwd): + if s3fs is None: + raise Exception("Package s3fs unavailable but required for this file source plugin.") + props = self._parse_common_config_opts(kwd) + self._bucket = props.pop("bucket") + assert self._bucket + self._props = props + + def list(self, path="/", recursive=True, user_context=None): + fs = self._open_fs(user_context=user_context) + if recursive: + res = [] + bucket_path = self._bucket_path(path) + for p, dirs, files in fs.walk(bucket_path, detail=True): + # print(p) + # print(dirs) + # bucket_path = bucket_path + "/" + p + to_dict = functools.partial(self._resource_info_to_dict, p) + res.extend(map(to_dict, dirs.values())) + res.extend(map(to_dict, files.values())) + return res + else: + bucket_path = self._bucket_path(path) + res = fs.ls(bucket_path, detail=True) + to_dict = functools.partial(self._resource_info_to_dict, path) + return list(map(to_dict, res)) + + def realize_to(self, source_path, native_path, user_context=None): + bucket_path = self._bucket_path(source_path) + self._open_fs(user_context=user_context).download(bucket_path, native_path) + + def _write_from(self, target_path, native_path, user_context=None): + raise NotImplementedError() + + def _bucket_path(self, path): + if not path.startswith("/"): + path = f"/{path}" + return f"{self._bucket}{path}" + + def _open_fs(self, user_context=None): + fs = s3fs.S3FileSystem(**self._props) + return fs + + def _resource_info_to_dict(self, dir_path, resource_info): + name = os.path.basename(resource_info["name"]) + path = os.path.join(dir_path, name) + uri = self.uri_from_path(path) + if resource_info["type"] == "directory": + return {"class": "Directory", "name": name, "uri": uri, "path": path} + else: + return { + "class": "File", + "name": name, + "size": resource_info["size"], + # should this be mtime... + "ctime": self.to_dict_time(resource_info["LastModified"]), + "uri": uri, + "path": path, + } + + def _serialization_props(self, user_context=None): + effective_props = {} + for key, val in self._props.items(): + effective_props[key] = self._evaluate_prop(val, user_context=user_context) + effective_props["bucket"] = self._bucket + return effective_props + + +__all__ = ('S3FsFilesSource',) From 9831e66fca60ff879a2d8fc72d3a7629500d7645 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 24 Feb 2021 12:36:18 -0500 Subject: [PATCH 58/70] More flushing to file source posix tests pass consistently. --- lib/galaxy/files/sources/s3fs.py | 3 --- test/unit/files/_util.py | 4 +++- test/unit/files/posix.py | 5 +++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/files/sources/s3fs.py b/lib/galaxy/files/sources/s3fs.py index b7a3fd63d3e..3e5fa4924ef 100644 --- a/lib/galaxy/files/sources/s3fs.py +++ b/lib/galaxy/files/sources/s3fs.py @@ -29,9 +29,6 @@ class S3FsFilesSource(BaseFilesSource): res = [] bucket_path = self._bucket_path(path) for p, dirs, files in fs.walk(bucket_path, detail=True): - # print(p) - # print(dirs) - # bucket_path = bucket_path + "/" + p to_dict = functools.partial(self._resource_info_to_dict, p) res.extend(map(to_dict, dirs.values())) res.extend(map(to_dict, files.values())) diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index e85e9ba1743..cd1f2627102 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -64,7 +64,8 @@ def assert_realizes_as(file_sources, uri, expected, user_context=None): file_source_path = file_sources.get_file_source_path(uri) with tempfile.NamedTemporaryFile(mode='r') as temp: file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) - realized_contents = temp.read() + with open(temp.name, "r") as f: + realized_contents = f.read() if realized_contents != expected: message = "Expected to realize contents at [{}] as [{}], instead found [{}]".format( uri, @@ -92,4 +93,5 @@ def write_from(file_sources, uri, content, user_context=None): file_source_path = file_sources.get_file_source_path(uri) with tempfile.NamedTemporaryFile(mode='w') as f: f.write(content) + f.flush() file_source_path.file_source.write_from(file_source_path.path, f.name, user_context=user_context) diff --git a/test/unit/files/posix.py b/test/unit/files/posix.py index 9543b61d31b..b139b1014f4 100644 --- a/test/unit/files/posix.py +++ b/test/unit/files/posix.py @@ -89,8 +89,8 @@ def test_posix_link_security_allowlist(): def test_posix_link_security_allowlist_write(): file_sources = _configured_file_sources(include_allowlist=True, writable=True) - write_from(file_sources, "gxfiles://test1/unsafe", "my test content") - with open(os.path.join(file_sources.test_root, "unsafe")) as f: + write_from(file_sources, "gxfiles://test1/unsafe_dir/foo", "my test content") + with open(os.path.join(file_sources.test_root, "subdir1", "foo")) as f: assert f.read() == "my test content" @@ -305,6 +305,7 @@ def _write_file_fixtures(tmp, root): with open(os.path.join(subdir2, "d"), "w") as f: f.write("d\n") + os.symlink(subdir1, os.path.join(root, "unsafe_dir")) return tmp, root From 68e1c34f4e86a822db8d1d1af58c5ff525774d37 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 24 Feb 2021 12:37:09 -0500 Subject: [PATCH 59/70] Fix galaxy.files unit test file names so they are picked up by pytest. --- test/unit/files/{dropbox.py => test_dropbox.py} | 0 test/unit/files/{posix.py => test_posix.py} | 0 test/unit/files/{s3.py => test_s3.py} | 0 test/unit/files/{webdav.py => test_webdav.py} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename test/unit/files/{dropbox.py => test_dropbox.py} (100%) rename test/unit/files/{posix.py => test_posix.py} (100%) rename test/unit/files/{s3.py => test_s3.py} (100%) rename test/unit/files/{webdav.py => test_webdav.py} (100%) diff --git a/test/unit/files/dropbox.py b/test/unit/files/test_dropbox.py similarity index 100% rename from test/unit/files/dropbox.py rename to test/unit/files/test_dropbox.py diff --git a/test/unit/files/posix.py b/test/unit/files/test_posix.py similarity index 100% rename from test/unit/files/posix.py rename to test/unit/files/test_posix.py diff --git a/test/unit/files/s3.py b/test/unit/files/test_s3.py similarity index 100% rename from test/unit/files/s3.py rename to test/unit/files/test_s3.py diff --git a/test/unit/files/webdav.py b/test/unit/files/test_webdav.py similarity index 100% rename from test/unit/files/webdav.py rename to test/unit/files/test_webdav.py From 7f93730e54864345af38f0e8c3c5065816f6d9cd Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 3 Feb 2021 20:11:44 +0100 Subject: [PATCH 60/70] Add ml notebook --- .../interactivetool_ml_jupyter_notebook.xml | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tools/interactive/interactivetool_ml_jupyter_notebook.xml diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml new file mode 100644 index 00000000000..cbc72391368 --- /dev/null +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -0,0 +1,93 @@ + + + anupkumar/sample-docker-ml-jupyter:latest + + + + 8888 + ipython/lab + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations, + visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, + machine learning, and much more. + + Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to + do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. + + You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. + If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. + + You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. + + From c37d57f53b773f45398b86ca56ed37809501d4ab Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Thu, 4 Feb 2021 19:46:08 +0100 Subject: [PATCH 61/70] Add jupyter notebook ml --- .../interactive/interactivetool_ml_jupyter_notebook.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index cbc72391368..6e609f0f2c1 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -1,19 +1,20 @@ - + - anupkumar/sample-docker-ml-jupyter:latest + docker.io/anupkumar/sample-docker-ml-jupyter 8888 - ipython/lab + lab/ - $__history_id__ + $__history_id__ $__galaxy_url__ 8080 $__galaxy_url__ + Date: Fri, 5 Feb 2021 15:52:41 +0100 Subject: [PATCH 62/70] Add tool and default ipynb file --- tools/interactive/default_notebook_ml.ipynb | 49 +++++++++++++++++++ .../interactivetool_ml_jupyter_notebook.xml | 6 +-- 2 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 tools/interactive/default_notebook_ml.ipynb diff --git a/tools/interactive/default_notebook_ml.ipynb b/tools/interactive/default_notebook_ml.ipynb new file mode 100644 index 00000000000..7729f6e2e7a --- /dev/null +++ b/tools/interactive/default_notebook_ml.ipynb @@ -0,0 +1,49 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "uniform-equivalent", + "metadata": {}, + "source": [ + "# Welcome to the interactive Galaxy IPython Notebook for Machine Learning." + ] + }, + { + "cell_type": "markdown", + "id": "attended-evolution", + "metadata": {}, + "source": [ + "More help and informations can be found on the project [website](https://github.com/anuprulez/ml-jupyter-notebook)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "excited-composer", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index 6e609f0f2c1..4b80f54b407 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -1,6 +1,6 @@ - docker.io/anupkumar/sample-docker-ml-jupyter + docker.io/anupkumar/sample-docker-ml-jupyter:latest @@ -30,8 +30,8 @@ export PATH=/home/jovyan/.local/bin:\$PATH && #if $mode.mode_select == 'scratch': - ## copy default notebook - cp '$__tool_directory__/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb && + ## copy default notebook ml + cp '$__tool_directory__/default_notebook_ml.ipynb' ./ipython_galaxy_notebook.ipynb && jupyter trust ./ipython_galaxy_notebook.ipynb && jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' From c2ee81e4e65e16e8b3d163cfc5d81e463e8b9a36 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Mon, 8 Feb 2021 20:36:42 +0100 Subject: [PATCH 63/70] Update tool info --- tools/interactive/interactivetool_ml_jupyter_notebook.xml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index 4b80f54b407..50647d6b97d 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -5,16 +5,15 @@ 8888 - lab/ + ipython/lab - $__history_id__ + $__history_id__ $__galaxy_url__ 8080 $__galaxy_url__ - Date: Mon, 22 Feb 2021 18:26:24 +0000 Subject: [PATCH 64/70] update docker image --- tools/interactive/interactivetool_ml_jupyter_notebook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index 50647d6b97d..d5f24e9a2ce 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -1,6 +1,6 @@ - docker.io/anupkumar/sample-docker-ml-jupyter:latest + docker.io/anupkumar/docker-ml-jupyter:latest From 5d2bbb720a3a2eb256ae180e1c70e26b67e7c848 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Mon, 22 Feb 2021 18:38:50 +0000 Subject: [PATCH 65/70] update notebook name --- .../interactivetool_ml_jupyter_notebook.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index d5f24e9a2ce..b442b25f0a1 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -30,21 +30,21 @@ #if $mode.mode_select == 'scratch': ## copy default machine learning notebook - cp '$__tool_directory__/default_notebook_ml.ipynb' ./ipython_galaxy_notebook.ipynb && - jupyter trust ./ipython_galaxy_notebook.ipynb && + cp '$__tool_directory__/default_notebook_ml.ipynb' ./tensorflow_notebook.ipynb && + jupyter trust ./tensorflow_notebook.ipynb && jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && - cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' + cp ./tensorflow_notebook.ipynb '$jupyter_notebook' #else: #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) cp '$mode.ipynb' ./${cleaned_name}.ipynb && jupyter trust ./${cleaned_name}.ipynb && #if $mode.run_it: - jupyter nbconvert --to notebook --execute --output ./ipython_galaxy_notebook.ipynb --allow-errors ./*.ipynb && + jupyter nbconvert --to notebook --execute --output ./tensorflow_notebook.ipynb --allow-errors ./*.ipynb && #else: jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && #end if - cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook' + cp ./tensorflow_notebook.ipynb '$jupyter_notebook' #end if ]]> From c4a76dad327958f17a019cabb36954a751e4f582 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 24 Feb 2021 09:30:26 +0000 Subject: [PATCH 66/70] Add path to run script --- .../interactivetool_ml_jupyter_notebook.xml | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index b442b25f0a1..e56329cdee9 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -1,18 +1,19 @@ - + - docker.io/anupkumar/docker-ml-jupyter:latest + docker.io/anupkumar/sample-docker-ml-jupyter:latest 8888 - ipython/lab + ipython/lab $__history_id__ $__galaxy_url__ 8080 - $__galaxy_url__ + $__galaxy_url__ + true From 598bf1426a00c8f58c749f5438411117d72ee81a Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 24 Feb 2021 11:33:15 +0000 Subject: [PATCH 67/70] Add permissions --- tools/interactive/interactivetool_ml_jupyter_notebook.xml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index e56329cdee9..59277509fe6 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -27,8 +27,12 @@ ## change into the directory where the notebooks are located cd ./jupyter/ && - export PATH=/home/jovyan/.local/bin:/usr/local/bin:\$PATH && - /usr/local/bin/start-notebook.sh + export PATH=/usr/local/bin:\$PATH && + cp '$__tool_directory__/default_notebook_ml.ipynb' ./ipython_galaxy_notebook_ml.ipynb && + chown -R \$NB_USER `pwd` && + ##chown \$NB_USER ./ipython_galaxy_notebook_ml.ipynb && + jupyter trust ./ipython_galaxy_notebook_ml.ipynb && + /usr/local/bin/start-notebook.sh ##if $mode.mode_select == 'scratch': ## copy default notebook From 5eb474a4d2d0a27b9c5167bbbc18e95150a6c5b8 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 24 Feb 2021 18:35:03 +0000 Subject: [PATCH 68/70] Update tool --- .../interactive/interactivetool_ml_jupyter_notebook.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index 59277509fe6..30b5d5d4e78 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -1,6 +1,6 @@ - docker.io/anupkumar/sample-docker-ml-jupyter:latest + docker.io/anupkumar/sample-docker-ml-jupyter:latest @@ -17,7 +17,8 @@ Date: Thu, 25 Feb 2021 10:13:25 +0000 Subject: [PATCH 69/70] Update tool --- .../interactivetool_ml_jupyter_notebook.xml | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tools/interactive/interactivetool_ml_jupyter_notebook.xml b/tools/interactive/interactivetool_ml_jupyter_notebook.xml index 30b5d5d4e78..ddf39644787 100644 --- a/tools/interactive/interactivetool_ml_jupyter_notebook.xml +++ b/tools/interactive/interactivetool_ml_jupyter_notebook.xml @@ -27,13 +27,24 @@ ln -sf '$input' './jupyter/data/${cleaned_name}' && ## change into the directory where the notebooks are located - cd ./jupyter/ && - ##export PATH=/usr/local/bin:\$PATH && - cp '$__tool_directory__/default_notebook_ml.ipynb' ./ipython_galaxy_notebook_ml.ipynb && + cd ./jupyter/ && chown -R \$NB_USER:users `pwd` && - ##chown \$NB_USER ./ipython_galaxy_notebook_ml.ipynb && - jupyter trust ./ipython_galaxy_notebook_ml.ipynb && - /usr/local/bin/start-notebook.sh + ##export PATH=/usr/local/bin:\$PATH && + #if $mode.mode_select == 'scratch': + cp '$__tool_directory__/default_notebook_ml.ipynb' ./ipython_galaxy_notebook_ml.ipynb && + jupyter trust ./ipython_galaxy_notebook_ml.ipynb && + /usr/local/bin/start-notebook.sh + #else: + #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier)) + cp '$mode.ipynb' ./${cleaned_name}.ipynb && + jupyter trust ./${cleaned_name}.ipynb && + #if $mode.run_it: + jupyter nbconvert --to notebook --execute --output ./ipython_galaxy_notebook_ml.ipynb --allow-errors ./*.ipynb && + #else: + jupyter lab --allow-root --no-browser --NotebookApp.shutdown_button=True && + #end if + cp ./ipython_galaxy_notebook_ml.ipynb '$jupyter_notebook' + #end if ##if $mode.mode_select == 'scratch': ## copy default notebook From 360b78312eabf75a6771b045df2e90e6b2a8d26b Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Thu, 25 Feb 2021 12:48:29 +0100 Subject: [PATCH 70/70] Update default notebook --- tools/interactive/default_notebook_ml.ipynb | 49 --- .../default_tensorflow_notebook.ipynb | 398 ++++++++++++++++++ 2 files changed, 398 insertions(+), 49 deletions(-) delete mode 100644 tools/interactive/default_notebook_ml.ipynb create mode 100644 tools/interactive/default_tensorflow_notebook.ipynb diff --git a/tools/interactive/default_notebook_ml.ipynb b/tools/interactive/default_notebook_ml.ipynb deleted file mode 100644 index 7729f6e2e7a..00000000000 --- a/tools/interactive/default_notebook_ml.ipynb +++ /dev/null @@ -1,49 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "uniform-equivalent", - "metadata": {}, - "source": [ - "# Welcome to the interactive Galaxy IPython Notebook for Machine Learning." - ] - }, - { - "cell_type": "markdown", - "id": "attended-evolution", - "metadata": {}, - "source": [ - "More help and informations can be found on the project [website](https://github.com/anuprulez/ml-jupyter-notebook)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "excited-composer", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tools/interactive/default_tensorflow_notebook.ipynb b/tools/interactive/default_tensorflow_notebook.ipynb new file mode 100644 index 00000000000..8ce18a1227f --- /dev/null +++ b/tools/interactive/default_tensorflow_notebook.ipynb @@ -0,0 +1,398 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "several-election", + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "rural-scoop", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Num GPUs Available: 1\n" + ] + } + ], + "source": [ + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "binary-blast", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Num CPUs Available: 1\n" + ] + } + ], + "source": [ + "print(\"Num CPUs Available: \", len(tf.config.list_physical_devices('CPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "wanted-exception", + "metadata": {}, + "outputs": [], + "source": [ + "fashion_mnist = tf.keras.datasets.fashion_mnist\n", + "\n", + "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "qualified-realtor", + "metadata": {}, + "outputs": [], + "source": [ + "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", + " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "considerable-private", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAD4CAYAAACE9dGgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdDklEQVR4nO3dfXBc5ZXn8e+RLMm2LL9hYww4MRCTYJLFZB0gMJUhYSZAKjWGSUhBzTJODTVmd2EnTPEHhJ2tsDXFFpUNsKnJwI4JbJwqCOsJMDAMFV4cEkIyvBjj4LclNtjBxsavYBvbsqXus3/01dCydM+9UrfUfc3vQ3WpdU8/fR+3pMO9zz33eczdEREpqpZGd0BEpBZKYiJSaEpiIlJoSmIiUmhKYiJSaGNGc2ft1uFj6RzNXYp8pHRzgCN+2Gp5j4u/2Om795RyvfbV1w8/5e6X1LK/WtWUxMzsEuD7QCvwQ3e/PXr9WDo51y6qZZciEnjJl9X8Hrv3lHj5qY/lem3rzPXTat5hjYZ9OmlmrcDfA5cCc4GrzGxuvTomIo3hQDnnf1nMbJaZPWdm68xsjZl9K9l+q5m9Y2Yrk8dXqtp828w2mNkbZnZx1j5qORI7B9jg7m8lO34IWACsreE9RaTBHKfH851O5tAL3OjuK8ysC3jVzJ5JYne5+/eqX5wcCF0JnAmcCDxrZqe7p3eoloH9k4DNVd9vSbb1Y2aLzGy5mS3v4XANuxOR0VKvIzF33+buK5Ln+4F1DJInqiwAHnL3w+6+EdhA5YApVS1JbLDBwwH3MLn7Ynef7+7z2+ioYXciMhocp+T5HsC0voOU5LEo7X3NbDZwNvBSsul6M3vdzO43synJtlwHR9VqSWJbgFlV358MbK3h/USkSZTxXA9gV99BSvJYPNj7mdkE4GHgBnffB9wDnAbMA7YBd/S9dJDm4Q3etSSxV4A5ZnaKmbVTOY99vIb3E5Em4EAJz/XIw8zaqCSwB9z9EQB33+7uJXcvA/fy4SnjkA+Ohp3E3L0XuB54isp57lJ3XzPc9xOR5jGEI7GQmRlwH7DO3e+s2j6z6mWXA6uT548DV5pZh5mdAswBXo72UVOdmLs/CTxZy3uISHNxoKd+U3RdAFwNrDKzlcm2W6iUZM1LdrcJuBbA3deY2VIqVQ69wHXRlUkY5Yp9EWl+PoRTxcz3cn+Bwce5Ug9+3P024La8+1ASE5H+HEoFmitVSUxE+qlU7BeHkpiIHMUoDXoG2JyUxESkn8rAvpKYiBRUpU5MSUxECqysIzERKSodiYlIoTlGqUAz1yuJicgAOp0UkcJyjCPe2uhu5KYkJiL9VIpddTopIgWmgX1pHpbxy1jjbAWtx00N4+9dfHpqbOKDL9a076x/m41pS415z5Ha9l2rrJ9LpH4zTKS8vVFyHYmJSIGVdSQmIkVVGdgvTmooTk9FZFRoYF9ECq+kOjERKSpV7ItI4ZV1dVJEiqpyA7iSmDQJa41vH/He3jDeMm9uGF937YS4/aH0WNuBcHV6xhyKJ0lue3p5GK+pFiyrBi3jc8XiJFBL32xM8Gcb/zhzcYwe3XYkIkXljopdRaTITMWuIlJcjo7ERKTgNLAvIoXlmCZFFJHiqizZVpzUUJyeisgo0eK50kTCmiKy68Q2Xzw5jP/Z538Vxn+989TU2O87Tgjb+rgwzJg/+nwYP/3ud1JjvZvejt88Y86urM8tS+uUKenBUilsW9q3Lz1Yh6nGnI9Qxb6ZbQL2AyWg193n16NTItJYH7UjsS+6+646vI+INAF3++gciYnIsacysP/Rue3IgafNzIF/cPfFR7/AzBYBiwDGMr7G3YnIyCvWHPu19vQCd/8scClwnZl94egXuPtid5/v7vPb6KhxdyIy0ioD+5brkcXMZpnZc2a2zszWmNm3ku1TzewZM1uffJ1S1ebbZrbBzN4ws4uz9lFTEnP3rcnXHcCjQDwtgYgUQomWXI8ceoEb3f0M4DwqBztzgZuBZe4+B1iWfE8SuxI4E7gEuNvMwnPbYScxM+s0s66+58CXgdXDfT8RaQ59Ffv1OBJz923uviJ5vh9YB5wELACWJC9bAlyWPF8APOTuh919I7CBjIOjWsbEZgCPWmXepTHAg+7+sxreT0ZAubu7pvZHzv4gjH99Ujyn19iWntTYL1vi+cLe+fmsMF76d3Hffn9nV2qs/Nr5YdvjVse1WhNf2xbGd33hpDC+89+nF3TNyFiOc8qzb6bGbE99rtUNYaGQaWZW/UuweLCxcQAzmw2cDbwEzHD3bVBJdGZ2fPKyk4DqT2BLsi3VsP/F7v4WcNZw24tIc3KHnnLuJLYrT32omU0AHgZucPd9lj7p5GCBsIRXJRYi0k/ldLJ+VyfNrI1KAnvA3R9JNm83s5nJUdhMYEeyfQtQfQh+MrA1ev/iXEcVkVFTSu6fzHpkscoh133AOne/syr0OLAweb4QeKxq+5Vm1mFmpwBzgJejfehITET66SuxqJMLgKuBVWa2Mtl2C3A7sNTMrgHeBq4AcPc1ZrYUWEvlyuZ17h4OUCqJichR6nc66e4vMPg4F8BFKW1uA27Luw8lMREZQHPsy+iKlhfLmFLmg2+cF8b/fO4vwvibPdPD+Mnte1JjV5z4atiW/xDHf/DGH4bxA29NSo21dMafy7vnxUci7yyI/93eE0/VM2VF+p9ey8LtYdt9R9KnNyotq/2umMrVyY/OvZMicozR9NQiUng6nRSRwqrz1ckRpyQmIgNoUkQRKSx3o1dJTESKTKeTIlJYGhOToYvqvEbYeTeFt6XxxQlra3r/k4IJCA54e9j2/VJnGP/O3H8J4ztPT5+KJ2tx2B+uj6fq+SCoQQNo7Y1/puf9xWupsa9NfSVs+92HP5Maa/EDYdu8lMREpLBUJyYihac6MREpLHfozT8pYsMpiYnIADqdFJHC0piYiBSeK4mJSJFpYF+GJmPOr5G0/oPjw/juiRPC+Lu9k8P4ca3py6p1tRwK285u2xXGd5bS68AAWtvSl4Q74vF8Wf/9zH8O491ntIXxNouXfDt/bPraF1es/fOwbSdvhfFauWtMTEQKzSjp6qSIFJnGxESksHTvpIgUmzd0mHbIlMREZABdnRSRwnIN7ItI0el0Ugpjekd6HRfAWOsJ4+0Wr6+4tWdKamz9oU+GbX+3L65hu2TGmjDeE9SCtQbznEF2ndeJbe+F8W6P68iiT/WCGXEd2MowWh9FujqZecxoZveb2Q4zW121baqZPWNm65Ov6b+pIlIo7pUklufRDPKc+P4IuOSobTcDy9x9DrAs+V5EjhFlt1yPZpCZxNz9eeDotegXAEuS50uAy+rbLRFpJPd8j2Yw3DGxGe6+DcDdt5lZ6uCFmS0CFgGMZfwwdycio8UxygW6OjniPXX3xe4+393nt9Ex0rsTkTrwnI9mMNwktt3MZgIkX3fUr0si0lDH4MD+YB4HFibPFwKP1ac7ItIUCnQoljkmZmY/AS4EppnZFuA7wO3AUjO7BngbuGIkO3nMy1h30lrjua+8N71Wq3VKXP3yh5NXhfGdpYlh/P1SPM45ufVgamx/79iw7Z5D8Xt/qmNbGF9xcHZqbHp7XOcV9Rtg05FpYXxOx7th/LvbL0qNzRp79HW0/nov+kJqzF/617BtXs1ylJVHZhJz96tSQuk/BREpLAfK5fokMTO7H/gqsMPdP51suxX4S2Bn8rJb3P3JJPZt4BqgBPyVuz+VtY/iXIIQkdHhgFu+R7YfMbDOFOAud5+XPPoS2FzgSuDMpM3dZhafhqAkJiKDqFedWEqdaZoFwEPuftjdNwIbgHOyGimJichA+Qf2p5nZ8qrHopx7uN7MXk9ua+wbuD0J2Fz1mi3JtpBuABeRowypfGKXu88f4g7uAf6WShr8W+AO4C9g0EnMMo/3dCQmIgONYImFu29395K7l4F7+fCUcQswq+qlJwPpy0IldCTWDDIGF2xM/GOKSiw2X3NG2PZL4+OlyX7THR/NTx+zP4xH0+HM7Ngbtu2a0R3Gs8o7po5Jn2Zof2lc2HZ8y+EwnvXv/mx7vNzcXz/72dRY16d3h20ntgXHHvW4qOjgdbo6ORgzm9l32yJwOdA3Q87jwINmdidwIjAHeDnr/ZTERGQQdSuxGKzO9EIzm0flWG4TcC2Au68xs6XAWqAXuM7d44ndUBITkcHUqRo/pc70vuD1twG3DWUfSmIiMlCT3FKUh5KYiPTXV+xaEEpiIjJAs0x4mIeSmIgMNIJXJ+tNSUxEBjAdiclQWFt7GC93x/VSkWmrjoTxXaV4abHJLfGUNO0ZS5sdCerEzp+6MWy7M6OWa8WhU8J4V+uh1Nj0lrjOa1ZbXKu1qntWGH/ywCfC+DVffTY19pPFfxy2bf/Zb1Jj5vHPK5cmmissDyUxETlK7hkqmoKSmIgMpCMxESm0cqM7kJ+SmIj0pzoxESk6XZ0UkWIrUBLTfGIiUmjFOhILljazMXG9k7Vm5OuWOF7uDuaXKmfOFhLynriWqxbf/4cfhPHNvZPD+Ls9cTxrabNSMKXLi4cmhW3HtvSE8elj9oXxfeW4ziyyvxwvJxfNkwbZfb/puPWpsUf2/lHYdjTodFJEisvRbUciUnA6EhORItPppIgUm5KYiBSakpiIFJW5TidFpOh0dXJ4allfMavWyuOynYY6tOCcML75srgO7c/OTl+a793errDtawdnh/FJwZxcAJ0Z6zN2e3r93tYjU1JjkF1rFa0rCXB8UEdW8rgu8J2euG9ZsurntvQGa2L+STzX2eQfD6tLQ1KkI7HMin0zu9/MdpjZ6qptt5rZO2a2Mnl8ZWS7KSKjagRXAK+3PLcd/Qi4ZJDtd7n7vOTxZH27JSIN4x+Oi2U9mkFmEnP354E9o9AXEWkWx9iRWJrrzez15HQzdQDBzBaZ2XIzW95DPH4iIs3ByvkezWC4Sewe4DRgHrANuCPthe6+2N3nu/v8NjqGuTsRkcENK4m5+3Z3L7l7GbgXiC+viUixHOunk2Y2s+rby4HVaa8VkYIp2MB+Zp2Ymf0EuBCYZmZbgO8AF5rZPCq5eBNwbT06E9WB1WrMzBPCeM8pM8L4njPGp8YOnhAXBs77yrow/s0Z/yeM7yxNDONtlv65be45Lmx79vhNYfzne+eG8V1jJoTxqM7s/M70ObUA3i+nf+YAJ455L4zftOHrqbEZ4+NarB9+PL7g3uPxgNAbPfHQyd5y+nxkfzX3ubDto0wP43XRJAkqj8wk5u5XDbL5vhHoi4g0i2MpiYnIR4vRPFce81ASE5H+mmi8Kw8tFCIiA9Xp6mTKbYtTzewZM1uffJ1SFfu2mW0wszfM7OI8XVUSE5GB6ldi8SMG3rZ4M7DM3ecAy5LvMbO5wJXAmUmbu80sXpEFJTERGUS9SixSbltcACxJni8BLqva/pC7H3b3jcAGctSgNtWY2OFLPxfGj/+vb6XG5k3cEradO+6FMN5djpd8i6aFWXvopLDtwXJ7GF9/JC7/2Nsblxq0BqOwO47EU/HcsTFeHmzZOf87jP/N1sHmBvhQy7j03/Tdpbg842sT4iXZIP6ZXfux51Njp7bvCNs+cWBmGN+aMVXPjLa9YXx2287U2J92/S5sewyUWMxw920A7r7NzI5Ptp8EvFj1ui3JtlBTJTERaQI+pKuT08xsedX3i9198TD3PFjBZWY6VRITkYHyH4ntcvf5Q3z37WY2MzkKmwn0HRZvAWZVve5kYGvWm2lMTEQGGOHbjh4HFibPFwKPVW2/0sw6zOwUYA6QPm1xQkdiIjJQncbEUm5bvB1YambXAG8DVwC4+xozWwqsBXqB69w9npsdJTEROVodZ6hIuW0R4KKU198G3DaUfSiJiUg/RrEq9pXERGQAJbE0Fi/Ldu7/eCVsflHXmtTYQY+nPsmqA8uq+4lMGhMvz3W4J/6Yd/TEU+1kOb3j3dTY5RNXhm2f/8G5YfwPuv9LGH/zS/E0QssOpRdc7+yN/91XbvxSGF/x9qwwft7sjamxz3S9E7bNqs3rau0O49H0SAAHyum/ry92x/Vzo0JJTEQKTUlMRAqrYLNYKImJyEBKYiJSZJoUUUQKTaeTIlJcTbQcWx5KYiIykJLY4HqO72Tr1elznN066e/C9g/uOS81Nmvs0fOu9ffx9l1h/Kxxvw/jka6WuGbokxPjmqEnDpwcxn/x/qfC+My291Njvzp4Wtj2oVv/Zxj/5l/fGMY//+R/DOP7ZqfPMdDbGf+lTDxrdxj/m7P/JYy3W/ptd++X4jqwqR0Hwvjk1rg2MEtU19jVkr7MHUDrJz+RGrNN8bx5eahiX0QKz8rFyWJKYiLSn8bERKTodDopIsWmJCYiRaYjMREpNiUxESmsoa121HCjmsRaemD89vRP54l988L2p45LX6tvV0+8vuJTH3wmjJ887r0wPqk1vXbnE8F8XgAruyeH8Z/tPDOMnzguXn9xe8+k1Njuns6w7cFgXiuA++66M4zfsT1et/LyqStSY2e1x3Vg75fjdWzWZqzXub88NjXW7fH8cnsz6si6gt8HgB6P/7RaPf3vYHJLXIO27zPHpcZK22v/ky5anVjmakdmNsvMnjOzdWa2xsy+lWyfambPmNn65OvwZxUUkebinu/RBPIs2dYL3OjuZwDnAdeZ2VzgZmCZu88BliXfi8gxYISXbKurzCTm7tvcfUXyfD+wjsrS4guAJcnLlgCXjVAfRWQ0+RAeTWBIJ9BmNhs4G3gJmOHu26CS6Mzs+JQ2i4BFAO2dOuMUKYIiDeznXgHczCYADwM3uHs80lzF3Re7+3x3nz+mIx5kFpHmYOV8j2aQK4mZWRuVBPaAuz+SbN5uZjOT+Exgx8h0UURGlVOogf3M00kzM+A+YJ27V19vfxxYSGVJ8oXAY1nv1XqkTNfmw6nxslvY/ue70qekmTF2f9h2XtfmMP7Gwfhy/apDJ6bGVoz5WNh2XGtPGJ/UHk/l0zkm/TMDmNaW/m8/pSP+f0s0XQ3AK93xv+0/Tf9FGH+7N30I4Z8PnB62XXsw/TMHmJKxVN6qfentD/a2h20Pl+I/je7euGRnUkf8M/3c1PSpn95gZth251nB9Ea/Dpvm1iyD9nnkGRO7ALgaWGVmK5Ntt1BJXkvN7BrgbeCKEemhiIy+YymJufsLVOrfBnNRfbsjIo1WtGJX3XYkIv25a1JEESm44uQwJTERGUinkyJSXA7odFJECq04OWyUk9gHh2j55Wup4X98+oKw+X9b8I+psV9mLGv2xLtxXc++I/GUNNPHpy/hNTGo0wKY2hYv/zUpo95prMVLvr3Xm34nxOGWeMqZUuqF54p3D6dP8wPw6/KcMN5Tbk2NHQ5ikF1ft+fItDB+4ri9qbH9venT9ABs2j81jO/aOyGMd4+P/7ReKKUvpXfJCWvCtuN2pP/MWuJfldx0OikihVbPq5NmtgnYD5SAXnefb2ZTgf8LzAY2Ad9w93hSvxS5750UkY+IkZnF4ovuPs/d5yff120qLyUxEemnUuzquR41qNtUXkpiIjJQOecDppnZ8qrHokHezYGnzezVqni/qbyAQafyykNjYiIywBCOsnZVnSKmucDdtyZzDj5jZv+vtt71pyMxEemvzmNi7r41+boDeBQ4hzpO5aUkJiJHqdw7meeRxcw6zayr7znwZWA1H07lBTmn8krTVKeTp970r2H87te/nt72P78Rtr30hNVhfMW+eN6st4O6od8Gc40BtLXEU2CObzsSxsdm1Eu1t6bPCdaS8b/LckadWGdr3Lesuc6mdqTXyHW1xnNutdQ4dWhr8G9/ee/ssO2M8XHt3ycm7grjvR4fH3x+0pupsfs3nh+2nfF3v0mNbfK4JjG3+k14OAN4tDItIWOAB939Z2b2CnWayqupkpiINIE6Lp7r7m8BZw2yfTd1mspLSUxEBmqSqafzUBITkYGKk8OUxERkICs3yVJGOSiJiUh/Tl8hayEoiYlIP0bNtxSNKiUxERlISSzQEswhVY7XQJz0wIupsd0PxLv96dcuDuPn3vJKGP/q7N+mxj7Vvj1s25ZxbD4243p2Z0tcy9Ud/MJlVTO/cGhWGC9lvMPP3zsjjL/fMy41tv3gxLBtW1D/lke0jumh3nietb2H4vnGWlviP/LuX8RznW1cmz7/3aQn49/FUaEkJiKFpTExESk6XZ0UkQJznU6KSIE5SmIiUnDFOZtUEhORgVQnJiLFdiwlMTObBfwYOIHKQeZid/++md0K/CWwM3npLe7+ZOYeM2rBRkrnwy+F8dUPx+1Xc0pqzD73J2HbQyek10oBdOyO5+Ta//G4/cQ30+eQajkcL0RY/u26MJ7tgxra7guj8SxqtWnPiE+veQ+/q/kdGsYdSsU5n8xzJNYL3OjuK5IZGl81s2eS2F3u/r2R656INMSxdCSWrETStyrJfjNbB5w00h0TkQYqUBIb0hz7ZjYbOBvoOze73sxeN7P7zWxKSptFfcs59RCfNolIE3Cg7PkeTSB3EjOzCcDDwA3uvg+4BzgNmEflSO2Owdq5+2J3n+/u89voqL3HIjLCHLyc79EEcl2dNLM2KgnsAXd/BMDdt1fF7wWeGJEeisjocgo1sJ95JGaVZUruA9a5+51V22dWvexyKsswicixwD3fownkORK7ALgaWGVmK5NttwBXmdk8Knl7E3DtCPSvEPyVVWE8ntQl28T0FboyFef/p9JUmiRB5ZHn6uQLMOjihNk1YSJSQM1zlJWHKvZFpD8HNBWPiBSajsREpLiOvduOROSjxMGbpAYsDyUxERmoSarx81ASE5GBNCYmIoXlrquTIlJwOhITkeJyvNSYyUuHQ0lMRPrrm4qnIJTERGSgApVYDGlSRBE59jngZc/1yMPMLjGzN8xsg5ndXO/+KomJSH9ev0kRzawV+HvgUmAuldlv5tazuzqdFJEB6jiwfw6wwd3fAjCzh4AFwNp67WBUk9h+3tv1rP/091WbpgG7RrMPQ9CsfWvWfoH6Nlz17NvHa32D/bz31LP+02k5Xz7WzJZXfb/Y3RdXfX8SsLnq+y3AubX2sdqoJjF377ecn5ktd/f5o9mHvJq1b83aL1DfhqvZ+ubul9Tx7Qabi7Culz41JiYiI2kLMKvq+5OBrfXcgZKYiIykV4A5ZnaKmbUDVwKP13MHjR7YX5z9koZp1r41a79AfRuuZu5bTdy918yuB54CWoH73X1NPfdhXqB7pEREjqbTSREpNCUxESm0hiSxkb4NoRZmtsnMVpnZyqPqXxrRl/vNbIeZra7aNtXMnjGz9cnXKU3Ut1vN7J3ks1tpZl9pUN9mmdlzZrbOzNaY2beS7Q397IJ+NcXnVlSjPiaW3IbwO+CPqVx+fQW4yt3rVsFbCzPbBMx394YXRprZF4APgB+7+6eTbd8F9rj77cn/AKa4+01N0rdbgQ/c/Xuj3Z+j+jYTmOnuK8ysC3gVuAz4Jg387IJ+fYMm+NyKqhFHYv92G4K7HwH6bkOQo7j788CeozYvAJYkz5dQ+SMYdSl9awruvs3dVyTP9wPrqFSON/SzC/olNWhEEhvsNoRm+kE68LSZvWpmixrdmUHMcPdtUPmjAI5vcH+Odr2ZvZ6cbjbkVLeamc0GzgZeook+u6P6BU32uRVJI5LYiN+GUKML3P2zVO66vy45bZJ87gFOA+YB24A7GtkZM5sAPAzc4O77GtmXaoP0q6k+t6JpRBIb8dsQauHuW5OvO4BHqZz+NpPtydhK3xjLjgb359+4+3Z3L3ll0cJ7aeBnZ2ZtVBLFA+7+SLK54Z/dYP1qps+tiBqRxEb8NoThMrPOZMAVM+sEvgysjluNuseBhcnzhcBjDexLP30JInE5DfrszMyA+4B17n5nVaihn11av5rlcyuqhlTsJ5eQ/xcf3oZw26h3YhBmdiqVoy+o3JL1YCP7ZmY/AS6kMlXLduA7wD8BS4GPAW8DV7j7qA+wp/TtQiqnRA5sAq7tG4Ma5b79AfArYBXQN3PfLVTGnxr22QX9uoom+NyKSrcdiUihqWJfRApNSUxECk1JTEQKTUlMRApNSUxECk1JTEQKTUlMRArt/wPRvxJhCQDpEAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.figure()\n", + "plt.imshow(train_images[0])\n", + "plt.colorbar()\n", + "plt.grid(False)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "extraordinary-description", + "metadata": {}, + "outputs": [], + "source": [ + "train_images = train_images / 255.0\n", + "\n", + "test_images = test_images / 255.0" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "controlling-paste", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj0AAAI8CAYAAAAazRqkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAACnYElEQVR4nO2dd7xcVdX+n0VReiAFCCGF0EkICQm9FxGQIgIiSpNXsbwqqD8QUXgVCyKigAooqAgYpUYBgYQWWogkgZBCCZACIUAICSU0Kfv3x8zdefbKnJ25N7fMvef5fj75ZJ05e86cOfvsPeeuZ621LYQAIYQQQoiuzgodfQJCCCGEEO2BHnqEEEIIUQr00COEEEKIUqCHHiGEEEKUAj30CCGEEKIU6KFHCCGEEKVgpeY07tmzZxgwYEAbnYqoxezZs7FgwQJr7eM2Sl++++670X7uueeivc466yTtVltttWibWU3bH2/RokXR/vjHP560W3/99aO94oorNve0W8ykSZMWhBB6tfZxO6o/P/jgg2R7wYIF0e7Ro0e0V1555eX+rLfffjva3M9Aer/4e6Kt6Apj87333ov24sWLk32vvfZatHmMcL8C6dgsGn8A8Oabb0Z7hRWW/L3dvXv3pF2vXq0+POqiLcZmo8yzbcn7778f7dYY561Bri+b9dAzYMAATJw4sXXOStTFiBEj2uS4rdGXXOOppT80TzzxRLS/8Y1vRPuzn/1s0m7YsGHR/tjHPhbtlVZKb+Hp06dHe9SoUdEeOHBg0u60006L9tprr93Ms245ZjanLY7bUWNz/vz5yfYVV1wR7eOOOy7a/JDZUiZPnhztJ598Mtl3+OGHR7u9Jt5GHpv1MmvWrGjfe++9yb5//etf0eYHk2OPPTZpt+2220ab++WGG25I2t15553RXn311aN9zDHHJO1OOumkus69tWmLsVmG38x58+ZFe4MNNujAM1lCri8lbwkhhBCiFDTL0yPKR86bU+TdefTRR5Pta665Jtr+rz92m7N7/YwzzkjaLVy4sM4zXsJmm20W7cceeyzZd84550SbvRCf/OQnk3bf/e53o7311ls3+xy6ItxPN910U7LvyiuvjPY//vGPaHvJgr117JnxEgvLL88//3y0P/3pTyft+D468sgjs+dfNm677bZo/+Y3v0n2rbrqqtH+73//m+xbZZVVoj179uxof+5zn0vavfzyy9FmKcd7YXv37h3tbt26Rfv6669P2l1wwQXR3nfffaN90UUXQRSz9957R9tLiz179oz2ZZddFu16pTf25gDAXnvtFe133nkn2v369UvajR49Otrs3etI5OkRQgghRCnQQ48QQgghSoEeeoQQQghRChTTI7LksrLeeOONaHOmjo+f4bigNdZYI9nHMQWcduzTyDk1+vXXX482p8v69+XOffvtt482p9mOGzcuaTd27Nho77rrrsm+q6++uvD4XRnuQ47NAIBf/OIX0f7Zz34WbZ9txXEgHLfjM+nWXHPNaHN8x4EHHpi087FAZefZZ5+N9siRI6Pt49I4HuOjjz5K9nFaed++faO91lprFX4ujzk/hvl9HMflY3922mmnaM+dOzfaHF8HAOeff37heZQR7j8uHQEAL7zwQrT5HvDz8RFHHBFtnt8+/PDDpB3He/GY5bIEQOPE8TDy9AghhBCiFOihRwghhBCloEvJWyyjAMXyhnfBPfDAA9E+4IAD6jo+u/u8e7Ze/Pky7VVVdnk47LDDos3VlNdbb72kHX8X7yYtqobs2/G14oqwvl3Re3KwxMZuWyA99/vvvz/Zx4UVt9xyy7o+q6vB0hSQurr/93//N9q//e1vk3ZcITsnbw0fPjzaX/ziF6PNKdRAx1XxbVRY+sldG5ZEfJVrHps8x2200UZJO5Y4+Rh+DvP3Sq1jA2mFX06pnjZtWtLulltuifZBBx1U89hlggtIctFJIJ0zufzHSy+9lLTjccphClOmTEnacSgC95ev1t2IyNMjhBBCiFKghx4hhBBClIIuJW/57AN2zz7zzDPRvvzyy5N2LG9wtLmXOjjjJydpsaziz4n35Y6Rk206ikmTJiXbLGlxxU+/CCXD2SJAmlWQyyTha8XXhjNMPFxh1q/HxFlBG264Yc3P8fjP4vuorJkkfB2BNGukf//+0fbXh/v9lVdeibavEMv3FR/b32P1Spll4YQTTog2V2H2UhdL0V72L1rDjKtpA2n/MT7Ly2daFsHH50VPeZwCkrQ8G2+8cbTHjx+f7OPfQr/4chE8Fr20z2ts8bzNiwI3KvL0CCGEEKIU6KFHCCGEEKVADz1CCCGEKAVdKqYnlw599913R/uOO+5I2nG1UU6r9PrkmDFjov3lL3852rkU7aKUbCCtIuvjRerVv9uTe+65J9nma8Wpqv67cHyO15N/+ctfRptXYeY+AdJVfrmdj/3hOASO6fEVex955JFo8+rNPuaB0zH99+IV48sa05O7v1999dXCfRyrw6vc+zHHsT+5atudocRDe8Lxh1zh+F//+lfSbocddoi2j5PivuB0aB/Tw2OG4yB9X/JY4jT3+fPnF3yLNF6Eq32LpeGyGX5e5PHBcau+L31qehM+vpVj6Lhfc9W6GwV5eoQQQghRCvTQI4QQQohS0KXkLe+qYyZMmBBtX82VXYFs77fffkm7Rx99NNqnnXZatEeMGJG04wXdfKXehx9+uOY57bzzzkm7Jpd0I6WuX3/99ck2yw183XzaN7u5/QKVLBOyfOjT40888cRo/+EPf4j2oEGDknYss/G1W3fddZN23/72t6N98cUXR5tdtf54fvE8XkRzxowZ0d5ss81QFnJV0Pn+8PcxpyK35LO8nJUrk1B2vvWtb0X7ggsuSPZxWQEv7fL9znJ7TsLgfvDH4305SYQXFOYK+Z1BOulIcqU3ePyx7M+hAgAwbNiwaPP19uUCvHzWhJ/fGxF5eoQQQghRCvTQI4QQQohS0OnlrZzLm7O0Jk6cGG3vJn3rrbeizTIF2wCw3XbbRXuTTTaJts8MGjduXLRvvPHGZB+7HTnD4rLLLkvaNUl1jVThkhegA9IMK3afFi0sCKSua88nP/nJaK+xxhrJPl7c81e/+lW0edFTALj55pujze50dtsCafYW94m/3pyx5bO3+Ps/9NBD0S6TvOXvfe57zvjw8hZfS96Xq6xcJEMDSy+WWXb43uf7+8EHH0za/eAHPyg8BktanBXpq6pzRXvuS9+OMzeL5BG/7+CDDy5sJ1JYqvLVtHlcsezs23G4AEuQvr9YxuIxn+vXRkGeHiGEEEKUAj30CCGEEKIU6KFHCCGEEKWgU8T0tHQF5TPPPDPaL774YmE7juPIrUb7wAMPRJtjhHws0bbbbhvtTTfdNNnHx//d734X7ZkzZybtmqr9+lWs25upU6dG26egFqUk+/gN1va5sqtn+vTp0fbXnvuP4xD8vcEaNe/jmBsPa+Fc+RnIVwHmWIb77rsv2scff3zhZ3U1cquds+21/pa049gU366RSjs0Aj5luQmfojxw4MBoz5o1K9nHMVk8D/nYNm7H/eLj8ng19lxf9uvXr+a5izw8P/uyLFtssUW0ub/8/OlLdjSRixHi+yFXNqZRkKdHCCGEEKVADz1CCCGEKAWdQt5q6WKC66yzTrRZHmFZAkhT7ti959Nx2S3Iko0/P5bBOH0dSN2CL7/8crT333//gm/RsZx77rnR9imoXLE1l/bN1827SVkm5AUqFy5cmLTjfuHr5o/Hn8WVR30F4GuuuSbaixYtira/N/h9fh+fk68gXRa8NMFpziw55WSr3KKlRWPfy5+iZXA/+PmOZQueI73kzuOMx19O6sj1ua+eLuqDF+71FC0Qmksx57HnZWze5nHOv7mNijw9QgghhCgFeugRQgghRCnQQ48QQgghSkGniOlpKRxbkosv4FgN1kV79OiRtOM0QNa7fdpfrhQ7v4917blz59b+Eh0Mr/7OsTQA8Mwzz0Sbl5fwMT2ctu/TXXfYYYdo8/Xw7Xib+8+nWBalOPuUZl6KhJeN4CVJ/Gf5ft5ggw2i/elPfxplJBcTwNfc92duPBbBcQQ+psffm2IJfH19P/Tp0yfaU6ZMKXwfX29/DF4ChPf5pUF4nuXYnwULFiTt/IreTfi4kqK0fJFe3+bAcTxs+xgsvvY8L/olnhoReXqEEEIIUQr00COEEEKIUtAp/INeVmC3K7vdfMolV9dl96xPpeSUS27HKdlAKuGw9OXlHD6er0r6xhtvRHvrrbeOtpdVmlK5O3qV9a9//es1bSBN9X766aejfckllyTtxo4dG21fkZmvwdprrx1tvoZAy1bvzVX6Zfcv9+uQIUOSdiNHjmz253Z1uN+9bMjXnN3jLV19meUSlje8+57HCcsqLXXzl4UBAwZE2/clj0Hu8/79+yftWOrgshM+fZnb8Rzs53fJVstPvWVefLui8evb8Xjmff43sxGRp0cIIYQQpUAPPUIIIYQoBZ3Cj+hda+yGZXmLq+wCaRVmXozNZ1TxMVhmeu6555J2XP2XK5R6dyxnFPnP4kyF//3f/4325MmTk3ZNrvyWLrbaHrD7evvtt4+2z6y5++67o+37kq8jX3ufqeEzRprw16doITz+HCDtS5ZDOFtN1Ib71/d1S93qTeSkbMZLMd26dYu2JK364QrauSrJRdmTQHH2lpe3eMFRH4rAeGlbNJ96fzd8O553c9mv3M9sz58/v1nn2RHI0yOEEEKIUqCHHiGEEEKUAj30CCGEEKIUdIqYHh/fUbR67+DBg5NtjjfgOBuvT7KWzZqkjw3gdGs+J18VmGNTvK7dt2/faHM69Kmnnpq023HHHQE0Vgqg13/5e3Of+HgNXpU5d+1z8SBFqZQtpShWhNPmPTlduzXOqbPA39Vfk/b6XB+jJYopiocD0rgNjnsE0jGdWz2bxwy/x8czrrfeetHm+J5GmuO6Ci2N6SlKRc/F/nB8JK9a0KjI0yOEEEKIUqCHHiGEEEKUglaTt9j9lVtMkNuxW6xeF2yOAw44INnmasi82F0uJZJdvF5W49TMIokNSM83t9AiL/DHKbeNipdwuP+YjTfeONnmRejqlSrrrRRaL7kq3EyuH/y9nEvx7crkJK1canNrvifXF7kFNstI7npwhXiuugykcyZXWvbwnMmVsbnSOVA81n1f+lIhTahSc/3k5K3cIspFx6i3bIzkLSGEEEKIBkEPPUIIIYQoBS32F+aycFrbDXnfffcl2zfccEO0H3jggWhzdVEgXRSUsz28q47Pl4/hvyMfg6Uuf7xcNgLLKtzuxhtvTNodfPDBhcdoFIoWfmW3OJBm0fF1A1KJjLPBvNu1KJOg3gq+uQUq+RhllayaQ+7eL+onf125n+rNAMu523mbx5iqM+clPpamBg0alOzr169ftHm8+Gv68ssvR5slLL8wKb+PZbXevXsn7V544YXC8xXFzJgxI9pevq938d/c3FrUjn8/ecWBRkWeHiGEEEKUAj30CCGEEKIU6KFHCCGEEKWgxcE39cY+LFy4MNmeN29etFmD5NeBNMaF2wFpjAjrkz6WhtMsN9hgg2h7TZpjSVif9itIs67Nq3G/+eabSbv7778/2l5P55RojmcZP348OhtFqeP+O+cqF+eqfha1aw1Nms+JY0py8Q9lqrqcI3eN6y0tUG/F2Ja8v960d5HOVb7UBMfk8JzJFdaBdP577bXXou1jLDnex8/3DM/BXCF/3XXXTdqpNEHKE088Ee0NN9ww2cfXnn/HPDwX5sYYt+PfyZdeeilpN27cuGjzb2ZHojtFCCGEEKVADz1CCCGEKAUtlrceeuihZPuss86KNi8mx+5OoLj6ql/okeUz705ldxq74HyqNLvTrrnmmmhvt912STtOn2Q3bq66JFdTXrx4cbKPXYtecmPXIi9M2hkqWbYUdmX7fi5KV87JJi3Bv5+lRd7nK0aLpWmNRUbrlTWL5DLfT3xO6sNi6ef5559P2j3++OPRHjhwYLKPKzRzqMAmm2yStON5bObMmdH2i5TyPJuDK+nzosynnHJK0k6SVspdd90VbS8t8/2QkwXrlaeLFib198Yll1wSbclbQgghhBDtiB56hBBCCFEKmi1vNbmRTz755OR1ljByC24WVSvmasdAKlV52YrhRe3mzJmT7Dv99NNrHoNdbkBaEZTlrb333jtpx9kNTz/9dLT9YnwsnXhXO7sF+Tr5zITOQL3ZTLlMP64cyvdKTt7KuWCL9vkKpSyR5mQTRtlbFXKVlotkq1xGVe66tiRrj+cEXuy2TBRJP6NHj062t9pqq2j7aul87Xhu7dOnT9LuySefjDbfDz6DiEMC1ltvvWj7+ZNlMa7OzHMuAGy66aYQS+AMYL8qAs9r9WZl5eCxyPeNz3jm7K1GQZ4eIYQQQpQCPfQIIYQQohTooUcIIYQQpaBZMT0LFizAX//6VwBLx89wuiOnMPpqxV6/bcLHUrAu77Vh1pTfeeedaLNODADHH398tP/5z39G269gPmvWrJrnPmnSpKTdPffcE+2iipRAGp/kY0kY1l19u6bU0tz7OwtFFbSBNAYgl0pZFHfD8VO+HfeRjxvxmncTvsSCWBquYO77syhewL++vPFRvv/4eD42RSyB42oAYMiQIdH2fclzj4+5ZIri4HJjmGMnfRo9xxIVxRUBiunxcNkTXy6g3lT03JxZBN83/HsMpBWa+R7yv5ntiTw9QgghhCgFeugRQgghRClolry18sorx9RqLzmxjMWuq379+hW2Yze5r9bZvXv3aPPCd/4Y7Cb1C4mydHLYYYdFe+utt07asVuQ5TfvguNqwiyr+LRdXtzNy1NFadne/d+0yGrOrdxZqHdx2pa4YItkKn+MnLzCfends0XvKTO59NeWuMfrJdfXRRW2RSrfc3kOIJUCuRIykPYzj+HcGMmVKymay/zCpCyJcCgDV/oXacVsIL0+vgQKX/uiVRGAdMzWW0KEj73ffvsl7a699tpoc7hIR1ZnlqdHCCGEEKVADz1CCCGEKAXNlreaZC3vuuzbt2+0OQPKuyRZIurVq1dNG0hdq94tyvvYPesX/mRXe48ePaLNi+wBqVuX5TgfAc+fxefr3e7savf72DXMbtxu3bol7SZPngwgXaC0s1Jvlc965ZB65YtcNV/ex677rnC925pcRmGRezxXTbkl+HuFxxzPPyLNjvLzNs+lvl95vuN5jMMSPCy5+LmvaFHYjTbaKGnHlZf5PZzRCwALFy6MNodDlIVHH320cF/udyc3LrnP+X7IVV7nsffUU08l7bj/nnjiiWhL3hJCCCGEaGP00COEEEKIUqCHHiGEEEKUgmbF9Ky22moYOnQogDQFHAD+8pe/RHuDDTaINq9MDqRp5RyD4/Vk1iC9hsx6MB/PVwZl3ZHTIn3aJmucrF3643E8UlGKvm/HNpCms7MWymmlwJLq0r7icCPRkpTklsZ2FMXx5OKFcinrRavd1xt/VGZ4rOYqXbd26jj3mY8x4HHy7LPPRnvYsGGteg6dEZ7H/PjjedHHs/G8y/OWv/Y8f/K86ONKeJ7k1dNHjBiRtLvvvvuizXO1n485fqiMMT233HJLst2zZ89o+98N7jPuLx8Hy2OWr7dvx5WyuZ85TtV/7tSpU2t8i/ZHnh4hhBBClAI99AghhBCiFDRL3mLOOOOMZLtJ9gKAX/3qV9H2sg2nerP046tyshvWp6wXpT7mqu7mUjNZSssdj+F9/tzZxctplUDqWmRXIC/8BwDHHHMMAOCCCy4oPIeOpt4Kyuwaz1VzZXxqbZG04d31/n1F58fnzserVy4rM/PmzSvcx/1RlL4O1F+5uWgRWj822cXObn6RVpn3cx/Px9OmTUv28Vjlkhr+GHztcyELHIrAC59+6lOfStrx7wIfw1cgLlrotCywjAukvzteZioq3+Lb3XzzzdE+6KCDor3qqqsm7VgK9ZW8i9pNnz69sF17Ik+PEEIIIUqBHnqEEEIIUQr00COEEEKIUtDsmJ4mjd1r9AceeGBN++67707acSwQr27uS4yzZu/jLDiVMpciyyvNctyAXyGetWbWJ+tNX+aYFSCN8fExJ5/4xCeiveWWW0a7I8tytyf+enA8Dfefb8fbRXEe/hiMjxspSp1Xyvqy4fHiy0nwdeZr6ful3jgqTr3ldr7fOZaEl5IR6VJA/r7n+I7XXnst2cfXm8uQ+FgdXq5n9dVXL/ysInxMCB+P7yc+NgC8+OKL0d58883r+qyuBMfcAMDYsWOj7ccbj5fcUjtF8Tm5pZZy7Xiu2HrrrQs/tz2Rp0cIIYQQpUAPPUIIIYQoBc2Wt4pSgovYe++9k+3x48fXbPfkk08m2+yS9audz507N9r9+/ePtpeZfDVo0brUm8LNrnFeQRlI3aF8b/n7jF3qvM+fA2/XuzI0o5T1ZbP99ttHe8aMGck+lkjYte1h9zv3U73XmKUNIL0nyih15OBV5315DZ8GzvCK2zy3+lRxnqs5Bd6vds/t2Pap10WlCfy9wSnaZeTLX/5ysn3SSSdF28tbLGP6itpM0e+7LwPB45zvjTfeeCNpx9snn3xy4ee2J/L0CCGEEKIU6KFHCCGEEKWgxRWZW5stttgiu80MHjy4rU9HtCLsCvUL17HsxJVjvczEmSD1SlW5hUQ5g48rz3pXe9E5AM2XersKLJEcd9xxyb577rkn2gsWLIi2lzpYIsktqsv9xv05YMCApB3L6F7CKTssKW+00UbJPpawPHy/c8aPly0583TkyJHR9jLYPvvsU/PYflzxfMF9OXDgwKTdXnvtVXjuZYSrXPsK/4xfIJuZP39+zdd95Wa+b3iMeslx9OjR0eZQlI6knLO2EEIIIUqHHnqEEEIIUQr00COEEEKIUtAwMT2i81HvKuvbbrtttAcNGpTs4xWVc7E6rPtz1dDc6ulF6fBAGkfCMQScju0pawyPh6+xj+844IADar5n4cKFyTbHCHA1dt+f66+/fk273nR4lRkALr744mj7irk8ro466qhkH8e3cTzG888/n7TjOKERI0bUdU6HH3544b4jjzyyrmOIFK547FPW77///mg/8cQT0fYrJuyyyy41j/2Nb3wj2ebYH75veDWGRkWzuBBCCCFKgR56hBBCCFEKrGiBxpqNzV4BMKftTkfUoH8IodeymzUP9WWHof7sOqgvuxat3p/qyw6jsC+b9dAjhBBCCNFZkbwlhBBCiFKghx4hhBBClIKGeOgxs8PMLJhZ8doTafvZZtazxuuLa7XPHKdZ7TPHOcHMNlh2y66PmfUws8nVfy+Z2Qu0/bHM+waY2bSCfWeb2b4F+5a69mZ2tJn9wMz2NLOda71PLBv1Zbkxsw+rfT3dzB4zs++YWUP8ZpQdjc2W0yh1eo4G8ACAzwH4UceeSos4AcA0APM6+Dw6nBDCqwCGAoCZ/QjA4hDCr5bzmGfVet3MVkTta78/gIsAHAxgMYBxy/P5ZUV9WXreCSEMBQAzWxfASADdAPwfNzKzlUIIHyz9dtFWaGy2nA5/ajezNQDsAuB/UHnoaXp9TzMba2bXm9mTZvY3c5XGzGxVM7vdzL5c47inmtkEM5tiZj/OfP75ZvaImd1lZr2qrw01s/HV944ys3WKXjezIwCMAPC36lP2qq1yYbowZjbIzB6uXq8pZrZpddeKZnZZ9S/LMU3X0syuqF7nJi/fWWb2ACoPy8m1r94jQwEsBPBVAN+u7tvNzPpX+3lK9f9+dPxLzex+M5thZge18yXptKgvy0EIYT6AkwB8wyqcYGbXmdnNAMaY2epm9ufqnPuomR0K1L4/qm3/bRXv0TQzOyr74aJFaGzWpsMfegB8GsDtIYQZABaa2ba0bxiAUwBsBWAgKg9HTawB4GYAI0MIl/EBzWw/AJsC2B6VjhluZrvX+OzVATwSQtgWwL1Y8hfMlQC+F0IYAmBq7vUQwvUAJgL4QghhaAjhHYhl8VUAF1b/ihwBYG719U0B/D6EMAjAawCKyra+G0LYNYRwNZa+9sMAPBZCmAXgUgC/qe67H8DvAFxZ7b+/ofJXShMDAOwB4FMALjWz4pK/glFfloQQwkxUfjPWrb60E4DjQwh7A/gBgLtDCNsB2AvAeWa2OmrfH/sDmBdC2CaEMBjA7e37TUqDxmYNGuGh52gA/6ja/6huN/FwCGFuCOEjAJNRuWBN/AvAX0IIV9Y45n7Vf48CeATAFqh0tOcjANdU7asB7Gpm3QCsHUK4t/r6XwHsXvR6vV9SJDwE4Awz+x4q9RSaHhRnhRAmV+1JSPubuabgdaAyod5WsG8nVFz0AHAVgF1p37UhhI9CCE8DmInKPSOWjfqyXLC3/Y4QQtP6IvsBON3MJgMYC2AVAP1Q+/6YCmBfMzvXzHYLIbwO0RZobNagQx96zKwHgL0BXG5mswGcCuCoqusMAN6j5h8ijUF6EMAB1DY5NIBzqk+eQ0MIm4QQ/lTHKaloURtglUD1piC7ESGEkQAOAfAOgNFmtne1aa6/mbcyH7cfgDF1nloosGttC6gvy4yZDUSlL5sWXuK+MwCH05zbL4TwRK37o+rVH47Kw885ZlYzlkQ0D43N+uhoT88RqLjB+ocQBoQQ+gKYhfTJsIizALwK4OIa+0YDONEq8UIwsz5WCcTzrFA9BwD4PIAHqn91LDKz3aqvHwvg3qLXq/abANas45xLSQhhFE2GE6uT58wQwkUAbgIwZDkOH6991Ru3UjXIL9lXZRyWxI19AZXg+SaONLMVzGxjVKTUp5bjnLos6styYpV4x0sB/C7Urmg7GsA3m/4INbNh1f+Xuj+skgX0dlU2+RWAbWscTzQTjc366OiHnqMBjHKv3YDKA0g9nAJgFTP7Jb8YQhiDinvtITObCuB61H4oeQvAIDObhIrH6ezq68ejoklPQSUmaFmvX4GKPqlA5vo4CsC0qit8C1RipVrKFahee1T+qrmT9t0MoOmvn90AfAvAF6v9dyyAk6ntU6g8xN4G4KshhHeX45zKhPqy67Jq9XpPR6UvxgAoSgr5CYCVAUyxSkr0T6qv17o/tgbwcPW1HwD4aZt9g3KjsVkDLUMhugxmdjmAy0MI45v5visA3FINShcNgPpSiMaks4/NRqnTI8RyE0L4Ukefg2gd1JdCNCadfWzK0yOEEEKIUtDRMT1CCCGEEO2CHnqEEEIIUQr00COEEEKIUqCHHiGEEEKUgmZlb/Xs2TMMGDCgjU6lmA8+SBfwfeONN6K9YMGCaK+44opJu1VWWbKsxworLHm+88d7660lhSdXX331aPfp0ydpx8doL2bPno0FCxbUqjq9XHRUX5adSZMmLQgh9Grt4zZif7755pvR/vjHP57s+9jHPlbXMd57b0nx2Lfffjva66yzznKe3fKjsdm1aIuxqb7sGHJ92ayHngEDBmDixInN+nCfHVZ71Yg88+fPT7bvvvvuaF922ZK1Rtdee+2k3ZZbbhltnnQXLVqUtHvooYeiveOOO0b75z//edJu1VXrqzvI37kl35cZMWLEcr2/iJb0pVh+zGxOWxy3NfqzKJOzpffwvffeG+2NN9442bfhhhvWdYxZs2ZFm7/fkUce2aJzak00NrsWbTE21ZcdQ64v26ROT70/+uylufDCC5N9d965pODju++mRRvZG/Pf//432hMmTEja3XjjjTU/d+WVV0622aPzn//8J9o777xz0q579+7R3mOPPaL9zW9+M2nXCH+FCtFceNzmvJpz586N9p///Odk3/nnnx9t9si2BnxOxx57bLLv3HPPjfbJJ5+Mevjoo48Kjy+E6JpolAshhBCiFOihRwghhBClQA89QgghhCgF7b721rPPPhvtgw46KNrrr79+0o6Dkn0MDmdpcYCyDyxcvHjxMt8DpHFBr7zySrR9lhdnktxxxx3RfvDBB5N2X/nKV6L9mc98BkI0IvXGtAwbNizZfvrpp6PNYwIAVltttWjzmPZxeRz3xmP9xRdfTNq988470eZEAn+8//f//l+0OQFhn332SdqNHDky2v778vVQfE8xPuC96Lrl4jlzyx+1JHB+3LhxyTbHYz711FPR3myzzZb7s7oyrZ3MUC/HHHNMtL/zne8k+7bddtto83zjf8frRSNbCCGEEKVADz1CCCGEKAVtIm/lXGHf//73o927d+9o+zRvlpb88VZaaclpszuO5SwgdX+xzXIWkBYnZCmNPwdIix2yS9cf7/e//32099tvv2TfGmusASE6inrT0nfaaadoT5s2Ldm33nrrRdvf+zxWeZ8fSy+99FK0WdLytbC4iCFLWjwW/TbPHX//+9+Tdlzg8J///Geyj69Ha9baKhP1XquWXNOxY8cm21OnTo02S64AcMYZZ0Sb+3LMmDFJu5ZKJI1Ivfdsrh1vc7t66+29//77yTb/nnJ/HXHEEUm7GTNmRNv/jvM4bY2xKE+PEEIIIUqBHnqEEEIIUQraPHvLZ2OwW3uttdaKtneLsTucXdJAKkd9+OGH0fZrb/E2u6595gcfn9vlssZYpvKudj6/m266Kdn3+c9/HkJ0FDn38KhRo6I9fvz4aPft2zdpx9KuH7d8/CIbSMc+u859RlmRHOfHMB+fx22/fv2SdqNHj472bbfdluw74IADCs+3DNQrYfjX/bxbxJVXXhltXu7n/vvvT9pddNFF0d5ggw2i/dhjjyXtOBOLM3wA4IILLoj20KFD6zq/zk6RNJVrx7+fHh6LPpOZZWhu538z77vvvmgfdthh0fZr722xxRbR5vAQjz9+S5CnRwghhBClQA89QgghhCgFeugRQgghRClo85ieRYsWJdsc08NasK/synE2XjPmVNiiNFMg1RpZx/T6JJPTRTnOiCs39+zZs/D8eLV4QDE9ov3Jxb0xXD2c7+k333wzaZerls4xPrkxx/vqrX6ca1c0D/iUej73Aw88MNnH8YdcTdqfu0+/F0t44oknou2vG6ecT5w4MdoLFy5M2h1//PHR3mOPPaLt43b4GGwDaczIM888E+1NNtkke/5dhXpj0nLzAe/LxdLw2Hv++eeTfTzG1lxzzWj7WKLzzz8/2n369En2tXb5CHl6hBBCCFEK9NAjhBBCiFLQ5n7aKVOmJNvs8mSpy6eq8rZPCec0xo033jjaAwYMSNrx4oecYrf66qsn7dh1xzIbV5AEgJtvvrnm8V577bWkHVeU5PR1ITqCIhf2oYcemmyz9MMlGWbPnl3YzktORW7wXGpsS/Cfy25v/r5+XuE5wc8rLL987nOfq3m8rky90oEvIcKLfbIs2K1bt6TdiSeeGO3f/OY30fZyBi84OX/+/MLz4zTnRx55JNnHC0JzP5dF3qp3MWHPyy+/HG2WHV999dWk3aRJk2q+x0ua3bt3jzbfG6+//nrSzi8W3pbI0yOEEEKIUqCHHiGEEEKUgjaXt9hNDAC77bZbtP/2t79F2y9qyAvGsRszh3e7vvPOOzVtLzlxdVeWvnym1TnnnBPt7bbbLtos0wGpC33mzJl1nbsQ7c1DDz1UuM9nUzI5V3muCjOTqxhbD/UulOjPlbPLfFXnCRMmRJvnrbJUZ/YSJF87vga5hZ15HvcLhP7hD3+I9u233x7tT37yk4XntO666xbuY+mLZRQAeOGFF6L95z//Odq77LJL0m7w4MGFx+/M5Pry2WefjfYpp5yStONQDc62mj59etKOQ0wef/zxaO+5555JO5YueU7xC73mMqrrpV4JXZ4eIYQQQpQCPfQIIYQQohTooUcIIYQQpaDNY3pOO+20ZJu1xb322ivaw4YNS9q98cYb0fYxPazZ82rNPXr0SNoVVY71Gj0fj1PpfJwRpztyPBKn9/rz8Npl2Wnp6r9F8QUtrZbLKZ31pnN6OD6EP7ezxIBw2QUgrV6cu47ch7mKzHyMnN6eSzEvul9yaeR8T/i0dI4r8KUrRo4cGW2uEFsWcmUAGH/fcB/dfffd0T7mmGOSdpdeeunynmICp1Hz7wUADB8+PNpcndnHqvlU7K5CroIyl3m54oorkn3+N7S59OrVK9nmuDmOnzrqqKOSdhwjlJv7eV9uxYQc8vQIIYQQohTooUcIIYQQpaDN5S2fjnjXXXdF+4Ybboj2mDFjkna86NzFF1+c7GMJiheT86mURTIIu+CB1P3JrjTvnuUUvl/84hfR9hLWOuusE+0bb7wx2cfVS32aZRmoV/rxrsui99Xr0vT30E9/+tNoz5s3r65jeHIu5EblscceizYvmgukFXTZLc3jw+/z8lHR4qZetuJ9uTT3osUGc4sL8z3h2/ECyH7cln0h0XrHJs+DALD77rvXtD1cNoTvm3pLG/h2vEAsz7lAGvZwwAEH1HwPAMyZM6fws8uAl7N4HPFYrneu45AVIP2N5z669957k3bf+973ol3vIqieeqVKeXqEEEIIUQr00COEEEKIUqCHHiGEEEKUgjYXsU8//fT0A0k35zS1LbfcMml30003Rfvss88uPD5rjV6jL4ob8Np9UbyPX66CU+B32GGHaPPqsUCqa/pVfcsYx5OjSLOvN76C04wBYPLkydG+7rrrou1jTzi18uijj4723//+97o+F0hTvH/5y19G+4c//GHdx2hv+F73cTYMx8f5VGbuM18ygPfx8X1sDccL8PFzKes5Pb+onU9/5fnCf6+5c+cWHl8UU29fMryvpavYc0yaLxtSdB/6uM+yx3HlYidzcTw87vkaHnfccUk7noP5szgWF0jjvXxJBIaXvPjf//3fZB8veZFDnh4hhBBClAI99AghhBCiFLS5b++www5LtjllfdKkSdHmtEIAOOSQQ6LNq+kCQL9+/aLNrlWfis4us1xFWHbP8Qrp3r335ptvRptTHX/zm98k7XifX2mYK0/7KtRdlVzaaVG66tNPP51ss5uUVwf3pQ4GDhwY7Q033DDaPs129uzZ0b711luLTj3LP/7xj2j/5z//adEx2ptHHnkk2izPAcUp4T5lnd3PXgIucon7fi6qsO0lJx63uUrcRePbv85zgq8eyxIJ9ydL2WJpiuQp/zrfN7n5ODdfMHzv/fWvf032HXTQQdH+/Oc/H20vg+WklDLQ0urxRVXs+boDaZo6r+DOJQWA9Lmgb9++yT7/DNEEl58A0lAHXjHBI0+PEEIIIUqBHnqEEEIIUQraXN564oknkm2Wjzjraccdd0zaPfjgg9GeOnVqso9dcrkMgaJKr7lFL4syEfz5sst06NChSbuNNtoo2t5Vt/nmmxd+diOSW5iT5REvgTA5Fyq7PM8444xoX3PNNUk7Xhyyd+/e0d5+++2Tdixxvv3229H2i9a+8MIL0T7zzDMLz4+lVX9O3/nOd6L95JNPRptlWyBd/LCj4XvfjwOWI+qtwOqPwe/jys1e6iiSrXJjk/H3FC8kyZWlfbYOy2L+O/IxLrjggmg3J6Ov0am30nlbk8uwK2rn4WrCPlRg4sSJ0f7KV74S7WeffTZpt/POOy/7ZLsY9cqHubmi3vuGf/84PGThwoVJu4MPPrjwGOutt160ecz66s/8u5BDnh4hhBBClAI99AghhBCiFOihRwghhBCloM1jeryGyvrt888/H21f1TiXOs5ph6w1+uqaRfE5uZWcOQ7Efy7Hd/D5+bgBjhfhmBUAeOmll6LN6dWNRE7LZXJxPAynI/Kqu0CaZsjVqgcNGpS04759/fXXo/3GG28k7TgFleOAWOMH0vuN0xvPO++8wuNtvfXWyT6OAeH4FZ8e30j4lF2maFVl3898T+TiMZhc7F295NLoeZzx+PZp+VxV3Z8TH5P7syvRUTE8OeqtyMzV1gFgm222iTZXVQeAW265JdqjR4+Otr8ffMxlGWjJPVCUor4sHnvssWgPGTIk2n61ey7/4ef0s846K9r8W/uJT3yiReckT48QQgghSoEeeoQQQghRCtpc3vLyCC/8yJKFlwRYZvKuNXZLs3vdf1ZRurVvV7RInneF8r6ePXuiCE7H85Vj582bF+1GlbfY/Vmv6/miiy6K9iWXXJLse/nll6Pt3cmDBw+ONt8P/J7c+eWkSu5XX33Xu1Cb8Cmso0aNKjyPn/70p9H+/e9/H+3+/fsn7a6++urCY7Q3P//5z6Pt5VveZunOp5dyqnC9KeatAY91L2/xfcrn7qu0s7zHcwyQStb//Oc/o90oad5dCe7L3Bxz7rnnRtvfh1/96lejfdVVVyX7+B498MADo82V2IH6JfqyUJTO7n/Hihbz9mOFFwHn3/jmzBs/+9nPos2/wUceeWTdx2Dk6RFCCCFEKdBDjxBCCCFKQZvLWz5Dokh+4IXJgHRhwJy8lXM111uRucit7116/LlcJZIlOyB1/fljcFXKRoEXoQSAO+64I9pPPfVUtH1GC0t1/L04QwZIF/7kzCsgvd5+H8PSA1/TnFTJ0oa/hzgri/vPLxzKVT794pp9+vSJ9mabbRZtL5tcdtllaBRmzpwZbXY9A2lfsLTr5Tr+fu0pbzG5Mcz3ope3ctXcWXIZMGBAzfeI1oHnSC85/ehHP4o2j/V11103aceZoJtuummyj/ud56nOKGfxvc73bG7s+fmupdlXRe8vGhMjRoxItrlqMmfR5fBhJTwueS7KhZjkkKdHCCGEEKVADz1CCCGEKAV66BFCCCFEKWjzmB4Pa7SsC/qKzD4uooiiGCH/WayFei2ft+td/ZfjIXKp8rkq0R3J/Pnz8bvf/Q4AcOONNyb7OJ4qVwWXdXOufuyvB1fR9H3EsTocC+Rjofhe4dgi/1kcl8L9wN/JH4M1ZF6hG0jvBx93xnEkfPxGi9viCuF8nl4TL6pG7vusqNI5UJzy6tOSvW5fBB+fj5FLjeXYMH/PcvyW7yceq88991xd59co+Hml3lITrf3Z3C++j3msP/HEE9E+9dRTk3YcH8dV+88///ykXS7Wiqs3cxzbTjvtVPietiZX+iC38nlLSoi0NrmYoM985jPR5qrLAPCXv/yl5nv8bzAf38/9HEs5bNiwZZ/sMpCnRwghhBClQA89QgghhCgFbS5v1Zvu6aUD7+JiiqoreympKLU9d058DO8y5s9imcCnaLPE4mmUhQx79OiBY489FgCw3XbbJfsefPDBaE+bNi3ac+bMSdqxPLBo0aJo+zRhvqbercmLuC5YsCDaOUmF3eb+s4rSOP1CmyzHsQTi3cd8r/jSBHwe7Lr3qeCf+tSnov3LX/6y5vm1Jffff3/N13OSE8tb/ntzZVwvHxW54ustLdFS+Jpz3/r7iKVWP8fw92yNBVLbk5zskUttbo1rXxQSwGMCSGXWX//619Hee++9k3ZcNuK6665r0Tnx98qdU3uSqx7fkn548sknk+0///nP0faSoa9I30ROZuLfKj8H/PCHP4z2K6+8Em0fKlFETi7LlajZeOONC99Xb/kMeXqEEEIIUQr00COEEEKIUtDu2Vv1wq4177otqlCZc0nn3IdFC456meK1116LNstbvhooZw54939HVbCtRdO58KKfALDDDjvUbO9lu1mzZkX7mWeeibavsMoVUb28V9SX3sXJCwjywnX8OpBKjZyJ5SVIdnPnXN4s+eT6jjOhWF4BOr6ir19YtAl/fxdVe+X7HkjlgpykXDSu/DafX+4a8+f6a1okx/nvzjKsl6/9d+kqtPb9l8tCyslsXGl5gw02iPaUKVOSdtdcc81ynmF677Fs3t4VmUMIUYLPVY/ne4+lIwC4/PLLo+2znBmej//1r38l+7iyftE5+HPkccRZdEAqO956662F58S/k1wFPyer8RgF0vtr1113LfwsyVtCCCGEEIQeeoQQQghRCvTQI4QQQohS0OYiNsdfAGnKaC4Gh7VAr8uzbpxLfSuqeOm1v6L0+Fw8Dp97v379knYTJ06Mto+baJSKzCuuuGKMc/Grh7/44ovRzumk3bt3j/aee+4ZbR+3UxRTAhTHafh7g49ZlL4OpCns/B6+74A0zTK3Kjefu79PuIIx3+c+NsSvUt7e7LHHHjVf97EeRTEGvi/4muTigvj4/trxNmv9/voXpUP74/E55SpG8/E7qrptW5CLs+GYrJdffjlpx2Odx3COemOE/u///i/Z5nuK43hGjRpV1/FyZUxyle85pqe9MbPs/FeLRx55JNnmPsvNkbwKPZcCAYCbb7452gcffHD2fGtx9NFHJ9v7779/tHNp5Dy26+Wll15KtjlGcuedd2728Tzy9AghhBCiFOihRwghhBCloE3kLZYcclUo11prrcJjsBs6l0rKx8+5xutNhc1JZ0Xu+gEDBiTt+Dxy7vVGwadY++0iWILMyQYsLfm096Lr4WXAokVhc+/j/vIya58+faLN94Z3oee+V9F9468fp+d2BP/+979rvu7lW95m+W+99dYrbOfHVdG9768dy2JFkhiQXuNcO+63XGXloj6rtd2ZyElOjz/+eLR96jHPwX6R55ZUL+aqy+PGjUv2sdxcVCU8R06OzbXtyMVjFy9ejPvuu6/meRxxxBHR5nuWJUcPl+HwqxiwlOTnoJNPPjnaOXmLOfTQQ6M9ffr0ZJ9PiW9NeMFgoP77UCnrQgghhBCEHnqEEEIIUQraRN7KLe7J7m+WGDy56qtFbk3v3irK2PLvL6oc6z+XZTbO+PEVmXPyViNVZF5e2J2ai9L3bljRvtx+++01X/eyMUtOfH9fcsklSbsvfOEL0fbyJC/syve+l9J4X26sF73HZwjyNrvHfeYaL5rrq3QX4TOevNzXFjTNE/VmSuWyt1oj46VevvzlL0d7xowZyb5bbrlluY6dq8zv4XvFL8zZnrz33nuYOXMmAOArX/lKsu/MM8+MNo8blgj9Ps4E81Ilvy+3aOdpp50W7S996UtJu+9973vRvueee6K97777Ju18JfzWxMt7PjShiHrHijw9QgghhCgFeugRQgghRCnQQ48QQgghSkGbV2T2Ohtri7lU3nqrqhaltNZ6XxP1rhKc04w5bmDQoEHJvtzK710ppkd0DrhMAOvjPkW5aLwcdthhyfa3vvWtaI8cOTLZx7FACxcujHbv3r0Lz4nxcRs8NjmewVfY5vftsMMO0eZUXQC49957ax671mc3cdNNNyXbHLfSVjR3ZfRce55zDjzwwGQfx4Gcfvrpyb7Pf/7zdX322WefHW2OHzvllFOSdltvvXVdx2sN+HfBr9rdnvTo0QMnnHACAOCPf/xjso9LCfA5+nHIK6vzfc+VtgGgZ8+e0fYxb3wPnHfeeTVtAOjVq1e0OU7zxz/+MYrg37hcGYF68d+r3ti7ej9bnh4hhBBClAI99AghhBCiFLS7vMVuttxCjJw+yy43IHXR56qoFi2amFvolM/Pu+CLFrDMpd7788stmidEW8BjkOWnet3Gnl/84hc17Rze3c7nwWPOzxe8zWnvuWru9ZKrJs0VcnmxRqDt5a0333wTY8eOBbB0qj/Pfbzgr6/Ay/Mnfxe2AeCZZ56J9vnnn5/s4zRlXsxyzJgxSbsLL7ww2rxoab33RkvJSXo8x/tFcTsKX7l//Pjx0eZFq/0iylwygb8Xp7ID6e9V7tpwCZHctWFZLSdNNleKBZb+bWUpzVdkLioR4ecUf28XIU+PEEIIIUqBHnqEEEIIUQr00COEEEKIUtAmMT1Fyz94cuWlWfPz2h2nrr766qvR9mX1600/Z1gz9XEDb731VrS5VLbXEvncfQyP12uFaGv+9Kc/RfvGG2+MNt/PQOunnjJ+jNSrv7c2HFfBK8kDaYwTzzm77LJLW59Wwn//+1/Mnj0bAOL/TcyfPz/aHBfFcyKQxm3wPNi3b9+k3THHHBPtIUOGJPvuvPPOaPOK6VOnTk3a7brrrtHmuCAfj8TzYlvH2XCMyCc/+ck2/ax6+f73v59s//3vf482Lynhf6v4d5J/k/w15Nga/7vD8Wp8fB/fyveUL0fBLO9ckfs99r/3RTE9udjcHPL0CCGEEKIU6KFHCCGEEKWgTeQtrobpXZz1Sk5HHHFEtN94441kH6ew82fl0te5XW41dnbVebmsW7du0R4xYkThZ7Gr2Z8Tn4cQ7QHLNrzKuF99m8dZvdV4c+TKRPB2LuW1aJ93qfN2LgV+//33j/bll1+e7OMyFJ/61KeizStPtwdcxbdeWOYHgLlz50abK2Pz60B6rfjeAFJJi+8NX9WZ7xUvnzHtmTrO8tavf/3raPPK5u2NT/vma8+VrM8666yk3YQJE6Ltfwtbm9122y3ae+21V5t9Tk4S4/sOKF65oSWp8oA8PUIIIYQoCXroEUIIIUQpaBN565133ol2zq3tFxZjfKR7Z4Ldbv77576zEG1NrvIrZ254GYThrC9fCZhhF3ZrZ4PlYAnZS9RDhw4t3Mfy1je+8Y22Obk2okePHtntssFZep2hL1l2ZdszY8aMaE+aNCnZN2XKlGjzQrJAKnHy75NfTeDSSy+t+bk+JGR5x3NO6jzttNOS7c0337xmOx86Uy/y9AghhBCiFOihRwghhBClQA89QgghhCgFbRLTw6v/brbZZsk+TmncYYcdCo+RS2dvaapae8EpnLNmzUr2DR8+vL1PR4gIj6vzzjsv2cfjtnfv3oXHaJRVq4vIzQ9c7oLTmoH0e7VnDJJoW37yk5909Cm0Gvx76n9bjz766Db73Nb+zc0db999963rGLkSNTk0soUQQghRCvTQI4QQQohSYPUuxAkAZvYKgDnLbChak/4hhF7LbtY81Jcdhvqz66C+7Fq0en+qLzuMwr5s1kOPEEIIIURnRfKWEEIIIUqBHnqEEEIIUQoa9qHHzD40s8lmNs3MrjOz1ZbRfqyZjajas82sZ/ucqagHM/uBmU03synVfi2uV9D8Y+9pZre01vFEHo3NrktbjFPu/+VpI5qP+nNp2qROTyvxTghhKACY2d8AfBXArzv0jCrnYqjEQn20zMYCAGBmOwE4CMC2IYT3qj96LVs4pZUxs5VCCB909Hl0MjQ2uyCNPE5F81F/1qZhPT2O+wFs4v+iN7PfmdkJuTea2Xeqf5FOM7NTqq+da2ZfpzY/MrPvVu1TzWxC9cn4x9XXBpjZE2Z2MYBHAPSt8VGimN4AFoQQ3gOAEMKCEMK86l/9PzazR8xsqpltAQBmtrqZ/bnaD4+a2aHV1weY2f3V9o+Y2c7+g8xsu+p7BprZcDO718wmmdloM+tdbTPWzH5uZvcCOLn9LkOXRGOz61A0Ts+qXvdpZvbH6sNl0zg618weNrMZZrZb9fVVzewf1X66BkCsAmlml5jZxKr34ccd8SVLhPqzBg3/0GNmKwE4AMDUFrx3OIAvAtgBwI4AvmxmwwD8A8BR1PSzAK4zs/0AbApgewBDAQw3s92rbTYHcGUIYVgIQSmIzWMMgL7VgXSxme1B+xaEELYFcAmA/1d97QcA7g4hbAdgLwDnmdnqAOYD+ES1/VEALuIPqT4EXQrgUADPA/gtgCNCCMMB/BnAz6j52iGEPUII57f2ly0LGptdjqJx+rsQwnYhhMGo/OAdRO9ZKYSwPYBTAPxf9bWvAXg7hDAElTHHZeh/EEIYAWAIgD3MbEgbfp+yo/6sQSM/9KxqZpMBTATwHIA/teAYuwIYFUJ4K4SwGMCNAHYLITwKYF0z28DMtgGwKITwHID9qv8eReWvxi1QmWgBYE4IYfxyfaOSUr32wwGcBOAVANeQF+DG6v+TAAyo2vsBOL3a/2MBrAKgH4CVAVxmZlMBXAdgK/qYLQH8EcDB1b7cHMBgAHdUj/NDABtS+2ta6/uVEI3NLkhmnO5lZv+pjru9AQyit9Uav7sDuLp6zCkAplD7z5rZI6j04yCkY1i0IurP2nSKmJ4mzOwDpA9qqyzjGLkFQ64HcASA9VH567Kp/TkhhD+4zx0A4K1ln7IoIoTwISoPMGOrg+346q73qv9/iCX3owE4PITwFB/DzH4E4GUA26ByH7xLu19E5X4YBmBe9RjTQwg7FZyS+rPlaGx2UWqM06+g8lf8iBDC89UxyH1ba/wCwFIF4MxsI1S8uduFEBaZ2RVY9n0ilgP159I0sqenFnMAbGVmHzezbgD2WUb7+wB82sxWq8ojh6ESgwBUJtPPoTK5Xl99bTSAE81sDQAwsz5mtm5rf4myYWabm9mm9NJQ5KuUjgbwTdKah1Vf7wbgxWqg6rEAeMW51wB8CsDPzWxPAE8B6GWVYD6Y2cpmxn/RiNZFY7OTUzBOm/7wWFC99kfUcaj7AHyheszBqPzIAsBaqDygvm5m66EijYo2Qv1Zm0b29CxF9cn0WlTca0+j4lLLtX+k+vT5cPWly6vuc4QQppvZmgBeCCG8WH1tjJltCeCh6u/tYgDHoPLUK1rOGgB+a2ZrA/gAwDOouFwPKmj/EwAXAJhSffCZXW17MYAbzOxIAPfA/YUfQnjZzA4GcBuAE1EZ0BdVf4RXqh5zeit+L1FFY7NLUDROX0Mlbms2gAl1HOcSAH8xsykAJqPaxyGEx8zsUVTG4EwAD7bq2QuP+rMGWoZCCCGEEKWgs8lbQgghhBAtQg89QgghhCgFeugRQgghRCnQQ48QQgghSoEeeoQQQghRCvTQI4QQQohS0Kw6PT179gwDBgxokxP56KN0YeQXXngh2m+9lRZc7dGjR7R79erVJucDAIsWLUq2FyxYEO211lor2uutt16bncPs2bOxYMGCXPXaFtGWfdnWvPvukkLMb7zxRrJvxRWX1CtcYYUlz/RrrLFG0m7llVduo7PLM2nSpAUhhFa/aTtzf3ZWNDa7Fm0xNtWXHUOuL5v10DNgwABMnDixdc7K4R9szjzzzGiPGzcu2XfcccdF++tf/zraiuuuuy7Zvvzyy6N9wAFLik+ecsopbXYOI0aMaJPjtmVftjVPPbVkdYrbb7892de9e/dor7LKkoroO++cLsjep0+f5T4PrnFVLZi3TMysTRbE7Mz92VnR2OxatMXYVF92DLm+lLwlhBBCiFLQoctQfPWrX432vffem+xjucvLR+wFuuiii6Ldt2/fpN2mmy5ZdqRbt27RXrhwYdKOPUn//e9/o+2lk969e0f7kksuifbNN9+ctLvsssuiPXDgQIj6qNdz8rWvfS3aDz/8cLLvgw8+iPZ7772HIr70pS9F+7HHHov222+/nbTbfffdo33++ecn+1ZdddVof/jhktUQWGITQgjROMjTI4QQQohSoIceIYQQQpQCPfQIIYQQohS0e0zP3XffHe1Zs2ZFe9iwYUk7jqfx6ezbbLNNtF955ZVoP/vss0k7zgjjTIspU6Yk7VZaacll6NmzZ+E5zZ8/P9obbbRRtF977bWk3Xe/+91ojxo1CqI+6o3peemll6K9zjrrJPs4JutjH/tYtH0fXX311dHmFHifyj59+vRo830CpPFk/Lkc6yOEEKJxkKdHCCGEEKVADz1CCCGEKAXtLm/dcccd0eZKlT69mGWG999/P9nHEhRLDiyPAGkaMcsUXn7gar1rrrlmtLkqNACsttpqNT9rww03TNqxNPfAAw8k+3bddVeI2rCMydWUgVQ+eu6556K9+uqrJ+04ZZ3lTV+RmWUxlllZEgPSfv72t79deO7+fIUQQjQemqmFEEIIUQr00COEEEKIUtDu8ta8efOizYt25uQtlql8W5YjvITBkgjjK+ayHMUVeVnO8sdnOcOfH2ceSd7Kw/KRz9JjOOuPZSuWI3PH8PcCH4PvJy+lDhkypOZ7gDSLbP311y88B0lfQgjRGGg2FkIIIUQp0EOPEEIIIUqBHnqEEEIIUQraPKbHxzdw/AyvfM42kFbJ9XDcBcfTLF68OGnH6csc++PjNvgc+T3+3Pl9q6yySuH5cUzPjBkzCtuJ9Fr5dHFmwoQJ0eb4mbXXXjtp99RTT9U8to/P4kreDMeZAcChhx4a7TFjxiT7hg8fXvOcfOkEIYQQjYE8PUIIIYQoBXroEUIIIUQpaHN5i6vdAqlk9M4770TbywpcMdfLUW+++Wa0uSKzT0tmmYHlMi8/cHo8y1u+HcslnIbspRPGV3UWKfUuMnrPPffUfN3LW5/4xCeiPXPmzMJjs7w1dOjQaE+ePDlpx/fU4Ycfnuzr379/zXPyJRFE/cyePTvZnjt3brRV7kEIsbzI0yOEEEKIUqCHHiGEEEKUgjaXt1588cVk++Mf/3i0WSLyUhJLB77iMVfh5ff57C2Wrfiz+HUglc94MVIvU3B2Ue/evaPtK/XyefTo0SPZx7JKr169UHa4b1mq9LBUxVWzx48fn7Tr3r17tPne8NmBe+65Z7RZQjn66KOTdj//+c8Lz6leaU7kue6666J95plnJvv233//aLOUOXjw4DY9p6uvvjram222WbJv++23b9PPFkK0HfL0CCGEEKIU6KFHCCGEEKVADz1CCCGEKAVtHtPz6quvJtscC/P6669H+7777kvafeELX4j2BhtskOzjOCFeIZvjcYDiCr8+doTbccq6b7fuuutGm2NJ/CraW265ZbS5AjUAPPnkk9FWTE9xevf999+fbM+fPz/aHM/h769FixZFm8se+ArMXEH5mWeeiTb3nWg+XJKCx4Uv3fCtb32r5r6BAwcm7aZMmRLtk046Kdrjxo2r63x8nN+f//znaC9YsCDZxyU01lhjjWj7+aerkivRkeOiiy6K9rbbbhttni+BdM7kuW/IkCFJuz59+tT1ufVyzjnnRHvQoEHJvkMOOaRVP0s0PvL0CCGEEKIU6KFHCCGEEKWgzeUtLytwNWWusuvbTZo0Kdq77757so9d3pzG6uUsdrVzmrqv3MySFldu9qnonEbPVZj/85//JO34GBtuuGGy77HHHov2brvthrJT5ELnlGEgdb1zf/mSACxxFlXa9u2YI488Mtn+zne+E+1f//rXheeu9PUKRYutLly4MNnmhWEHDBgQ7ZwkwnOEvz/22muvaN9yyy3RHjVqVNKOJSw//o4//vhot3VKfCPiS4MUlZC48847k+3Pfe5z0WbZyl97rnbO8+fFF1+ctGOJc7vttos2L/ALpFK0r+R91113RXvOnDnR5v4HJG/Vix/XfA9wf2288caF72uUeVGeHiGEEEKUAj30CCGEEKIU6KFHCCGEEKWgzWN6vvSlLyXbvAr2a6+9Fm1OewTS1FJO8waAVVZZJdocx+NjdThllpea8PokH4O1Zo4/AoCHH3442lw638d6cArupZdemuzjZTjKiI8bKEpZHzNmTLLNsTt8fXlJCiDt56KSBcDSqe5NHHvssYXnd+ihhyb7/vWvf0W7UfTq1oLj4fx3y33Xov7ceuutk21eLmT69OnR5jIDQBrHwX32zW9+M2nHsXPbbLNNtL/73e8m7ThWh8tneIpiyICll7HpTHC/Aukc6WN4nnjiiWjzfMfLtgDArbfeGm3uP3+d+vXrV/Oz/BIxvP38889He8KECUk7jh/y5/7Zz3422lziZMaMGeiqtEb8DC/3c/bZZ0eb4+4A4N577432wQcfHG2OgVye8yjid7/7XbSHDh2a7Nt1113rOoY8PUIIIYQoBXroEUIIIUQpaHN5y8Np3zfeeGNhO3ZD++q87MouSpH1sFvXu3hZcllrrbWi7SUQbsfu+Z/+9Kd1nYPIuzu5FIFPQd1oo42izVW4WeoEgL59+0abXbW+yquvot0E358A8OCDD0abq4R3BXJSR9H1aS3OO++8aO+zzz7RZskQSCsjszyy3nrrJe3Y7b3HHnss9/nxfdoZ5Cw/D/I220XyIwDcfvvtyfZvfvObaH/jG9+Itq+aXSQZvfzyy8k2X1OWpVdfffWkHd+XXFrC3698b/hSE3z/skTGFduBpaW6RqToN645sjPL/iwn33TTTUk7lgKZqVOnJtuc6s/X1P9Wt6QsC5erAYCvf/3rNc/j05/+dNJO8pYQQgghBKGHHiGEEEKUgjaXt7xrrkhm8i5kzvZgNyaQuvH4GD7LgiP6c+56fh8fmzO5gNRNmsNnKDE593IZyPUDZ2z5+4Gz3thV6/ucF5hkGcwvGsnVffmznnvuuaTdmWeeWXi+J5xwQrSvuOKKwnbtRdNYy7m5eTzm+uKll16K9lVXXZXsu+2226J99913N/s8AWCHHXaINmfa8LGBdAwXyR5Aml2Uk7d4bPKCx0B673Dl3nnz5iXtmjKUfOZgR+LnWe5bvm5cCRsANt9882j/+Mc/TvZxBi1Xp2epGQCOOeaYZp8vZ+6OHj062ceVm1mi9jIYV//1Ff1ZWuN+8vNKe8hbTX2TW9A1N2ZbkgHl57Ezzjgj2nw/sGQMpFlaHMKx5pprJu1YFuNVEXwVbl6tgDNwfT9whrY/91122SXaHPYwbdo0tAR5eoQQQghRCvTQI4QQQohSoIceIYQQQpSCNo/p8Xokx7TkYgp8HA/DlXZ5RXNflZP1+6I4IH8efDyvIecq/BYdr6tV6m0J3A8+ponjbrgqt6+2ybEIXHnb94nXnpvo2bNnsv3ss8/WPD8uWQCksTo+nX3s2LHR5pW9DzrooJrn0F74+7vee/CUU06JNlcf99eEU1Q5nRRYesXsevjDH/4Q7b///e/JPr7GrOf7aul//etfo82xd1wBHkhjON54441kH8eH8Vzi4w823XRTAGkMUHtRVHXXz6Xcf9xfnNoPAHvvvXe0//3vfyf7+Hpz3A7HT3mKrqGH40COOuqoZB9vc9zG73//+6TdHXfcEW2O8wPSOCyeL3zF7/agqZ/qHYd+/PJ9tmDBgmj72JeFCxdG++mnn072cSkPrljO8VNAOhfyWPbXbd9996157n4+5vHG49KvnsAxm1xpG0hjsg488MBo+5IIHHeWQ54eIYQQQpQCPfQIIYQQohS0e0Vmhl1p3hXK7kq/j93N7PrzaawsVfF7vPuQj8+pqt5Vt9lmm9X4FkvTGgu/dSVyafpczZrdn+z+BlL3bJHUBSwtSdZzTnw/eJmA7ymW4oC0GjQvuuhlk89//vN1ndPy0lw3umfQoEHR/tvf/hbtJjmniU022STaPkX19NNPj7ZPhy2Cxya73oHUxc7Xn9NYAWDYsGHR5nIXfqHE7bffvubxPDwn+Mrs6667LoD677WW0HRP1lt195JLLkm2WZrift1zzz2TdiwR+X0PPPBAtFlWyM2DfH65FO1650iWvH3pAP798HInj0GeS3zYhC9l0Zb4352iNG2WqYC0tAJLPV7KZ2nRX/utttoq2vfdd1+0OY0cSCudN93nwNJzGq+KwHiJicczlynwY4d/x30pCC6RwIvRsoQLpNJfDnl6hBBCCFEK9NAjhBBCiFLQofJWjhdeeCHaPnuCZSvGu9aKFgr0EkaRlJbL8uKodO/qq3cR1K5K7rp5ODuK3dC++jVnELF88cwzzyTtOFOFpQ2faVPvIpIsd3p3Mme+tCRrqTUJIUSpz7uH2SWckxK+/OUvR5uzqLzscdZZZ0V7xx13TPZxdV0+nu/P8ePHR5ur7vqxPWTIkGhvt9120fbucZaqOMtu4sSJSTs+D3a3A6mEyvewr9rbJPW0pXTd3AVf/RzEch/LHl6q5IWd/ffcdttta+7jTBtPvRXnc9eO76HLLrss2vvvv3/Sjhc69dmZXE2f739/fm0tby1cuBBXX301gFT6BYATTzwx2pyx5LMlWYLi7+mlOq5K7TOgWDLjzFh/P/B8x4vM+t+0osr3fjUCv8BrE/Pnz0+2WZryczN/1iOPPBJtvyh1vcjTI4QQQohSoIceIYQQQpQCPfQIIYQQohR0aExPTtd96KGHou01Pk5TZu3da82sT/I+r+tyO44V8Ct4czvWJL2ezufUlVdVr7c6LHPzzTcn2xwrwDE9fK2BNGWS01N9ijPfG3PmzIm215r5s/h8c1VkBw4cmGz/6U9/Kmzb3rz33nuxyrRftZr7KbdSOccIcGyNT0vndr6sw0knnRRtjiPwFXP5fVtssUXyPRiO45gwYUK0+/TpgyI4xXe33XZL9k2ZMiXa++yzT7KP70Ue+7wSObDkfmmkchQ+fbcolsJXseWyC77iOKeIcwXzHHzdXnzxxWQf9wvHbPpYTP7cG264Idq+BAJXCfYxXvybwfeaj3fLjffWYK211sIBBxxQ87O4z+pdMZzjCv0cOWvWrGj7z+Jxxe/zx+B5kvuS+86/j+dP/1vN455jlXx/8ZySG1f8O+7v5UmTJhW+j5GnRwghhBClQA89QgghhCgFHSpv5WQQTkXOyVEsZ3h5qygVPSc5sVuf0x798bgqMKd2Ao3l9m5LWvI9Od0ZSNPKOX3Spzhzv3CqIleNBdJqsXx/3XPPPUk7vh9Y5vEyTNE55MhVom0rVlhhhegiZrkISK8JV4H1qbHsLuZ0Wp/Wym70k08+Odn36U9/Oto8LnILDPLiiF5imTp1arRZkvQyGB+f+9AvvMjHuP/++5N9LJWyDOgrATdVqm0raWTx4sXxvr7xxhuTfb179442fxc/V7FkxPetlzQ5HfiJJ55I9vF9zOn8t99+e9KuaJFRL1sVyche6uD7l9/j54THH3882n7c8jZLLj5V+n/+53/QlphZ/PzPfe5zyT6/vbzwd/a/rTxe+Hr4uapojvO/mXwMtjvyt89X5S5Cnh4hhBBClAI99AghhBCiFLS7vFW0uKPPlOLqkl62yi1qxxRJX94tzccoWogSSN14LG95mltNtSuQW7STs24mT56c7OPKodzOLzjKi87xgpfepckVOzkjYNddd03acUVgvk98NhLfa1zZNUdHuHhXWGGFKF1wZgyQZlFxFlz37t2Tdpzxw/3iZQWu6MoLJQKppMXSFGfaAGkWClfF9VISu9s508jLW7zN96KvTMvZKb4/X3rppWjnFm9skpLaapyvuuqqsVKy70ve5oVQeaFIIJXB+Br6hSO5Eq6/pix98TXgRYKBVKLm7Cg/pzN8PH99+b7hPvL9xeMsJ0vzYpv+eh533HGF72sNVlxxxSgj+2vP23xfeimJf69y7Rg/B3Hf8jjyx/C/eU34Pir63fWv8/HY9vca3yu578XH8JI5L5Cao3y/zkIIIYQoJXroEUIIIUQp0EOPEEIIIUpBu8f0FGmBXu/klWV9miGn2nJMh68G6avwNuG1Zj4nfo/XRfl9fnVvhrX+jkhfbk2KNFkg/Z65+Ibvfe970WY9GUivB+/z2junqXM7Xy2X9XtOwebqzEC6ujSncXs9mWN8fFxKI8GxA74veLzkKphznA2PP79CPacK+3uCxyqnuvsxVxSD42O5OH2ZY5M4ZgVI+5C/l48d4LgQH9PEsS9c/ZePDSyJFWurausrrrhivA5HHXVUXe/xcx1/F04d933J197PwXzvc8yMn8N4tXo+nl/BnMct3w++SjIfj9vlVt/2fcH3PKfz++r5/h5oS3yJCL8t2gd5eoQQQghRCvTQI4QQQohS0DDylk+LZVdrLv2O09Z8O3bJFqW++vdxtWd29wNp6mCR6xdI3bDe/d+IC5D6PuHvw9+z3hTd8847L9nm9PA99tgj2Tdu3Lho87Xx6ans5ubz84saeim0icsvv7zwnDiN3ruc+bN8+nMjYWaxr/y14/IK3J9+UUpeVJDT/XNpqB6+XixHcWo0kI5hlqj9sfl4ubRk7je+T/39wfOMr2LMshjPCZyi74/fKPh5hascs11vWq8QXZXGG71CCCGEEG2AHnqEEEIIUQo6dMFRxmdI1Fs5NiczsSSSk7f4GJw54LMF+H18PJYFAKBnz57RzlWMbhS8LOirEjfhM0S4Gu9vf/vbaP/mN79J2u20007R5qq3ALDzzjtHm6sp+0rLRdJDTmq46aabon3wwQcn+2699daa7/HH4/7LVWTmdh2dofeZz3wm2WbJiBfg9H3B0uDMmTOj7ReE5HvfVzfna8TjjytqA2kmHMvIXqbhLC1+T70Sk79n+Tv68c2SW05qFUJ0XuTpEUIIIUQp0EOPEEIIIUqBHnqEEEIIUQoaJqaH01uBVF/3cQMcQ8OVY71+z7EVHNfgq8Nyei7H9PiUdT4Gf5aPjeCYns7I9ddfH+0vfvGL0fbXjWM7GB8DMX369GgPHz482TdlypRob7zxxtGeNm1a0q6oMqu/9qNGjYq2j+Nhiqp1e/ge8hVmGb43Gq0sAce/cAVrX826K5KLERJClA95eoQQQghRCvTQI4QQQohS0DAVmWfNmpVs+3RShheaGzhwYLT94oIMS2J+4UhO0eZjc3VmIE2bZjnDp1cznSFl3VetPfXUU6PN0iLLgDm8dMT98tBDDyX7dtxxx2hzmrT/LE415gUUDzvssKTdpz/96brOsSgt38shLA35xTCZztDPQghRduTpEUIIIUQp0EOPEEIIIUqBHnqEEEIIUQoaJmXdx1Lwkg+52BqO/eEV14E09oNT4n1JfP++JnxsCp8jL3mRW3YgtyJ1o8DLNQDptVp//fWjzdcTSK8Pp6/778xxMT72ZcKECdHecMMNoz1ixIikHS9RMXv27GjfeOONKIJjifieAZZeWqGJonsBANZbb73CfUIIIRofeXqEEEIIUQr00COEEEKIUtAw8pZPIWYpyUsO6667brRZOvESBr+Pj+dXbX/77bejzbKHl2KKZCy/ajtT72rQHclxxx2XbF977bXRfuKJJ6LN6fxAccXrXNr3qquumuzj9z377LPR5hR1IK2Ufc899yz9JWrgK3kzRSUR/Hu4EnQuZZ+lvtznCiGE6Dga/xdZCCGEEKIV0EOPEEIIIUpBw/jhZ8yYkWyznOGliEWLFtW0vQz26quvRvuNN96I9jPPPJO0e/nll6M9efLkaO+0005JO5Z3WPoqqu7bWfCS01133RXtuXPnRvuKK65I2v373/+ONmdX5TKg6sUvZnrrrbdGe88991zu42+66aY1X+f7Dkgrfg8aNKjweI22yKgQQoilkadHCCGEEKVADz1CCCGEKAV66BFCCCFEKWj3mJ6iFG5fgXfBggXR5hR1IE1N79WrV7R9XMW8efNq2sOHD0/aceXeOXPmRNunqK+22mrR5tgfrlrs6Qwp6zm4SvIPf/jDZJ/fbsLHZ/Hq6RyDBaTlAzh+pijmprXgleS32267aPt7jc+vR48ehcdTmroQQjQ+nfsXWQghhBCiTvTQI4QQQohSYL7qcLax2SsA5iyzoWhN+ocQei27WfNQX3YY6s+ug/qya9Hq/am+7DAK+7JZDz1CCCGEEJ0VyVtCCCGEKAV66BFCCCFEKejwhx4z62Fmk6v/XjKzF2i7cH0HMxtgZtMK9p1tZvsW7DvBzDZwrx1tZj8wsz3NbOfl+0blxswOM7NgZlvU2X62mfWs8friWu0zx2lW+8xxlro/RJ7q2JluZlOq43aHVjjmWDMbsbxtRPNQX3Z+2qIP6dh7mtktrXW8jqDDi4uEEF4FMBQAzOxHABaHEH61nMc8q9brZrYigBMATAMwj3btD+AiAAcDWAxg3PJ8fsk5GsADAD4H4Ecdeyot4gQsfX+IAsxsJwAHAdg2hPBe9QG2cy9GV1LUl52fRu5DM1sphPBBR59Hh3t66sHMBpnZw9Wn1ilm1lS5bkUzu6z6VDvGzFattr/CzI6o2rPN7CwzewCVH+QRAP5WPdaqVqlAOBTAQgBfBfDt6r7dzKy/md1V/cy7zKwfHf9SM7vfzGaY2UHtfEkaEjNbA8AuAP4HlYeeptf3rP4ld72ZPWlmfzNX+bHaF7eb2ZdrHPdUM5tQ7YcfZz7/fDN7pNpXvaqvDTWz8dX3jjKzdYper94zyf3RKhema9MbwIIQwnsAEEJYEEKYVx1zE8xsmpn9sam/q/fBudXxPMPMdqu+vqqZ/aPaH9cAiNfezC4xs4nVcV7Y/2K5UV92for6cLaZ/bg6P061qifezFY3sz9X+/dRMzu0+vqA6u/bI9V/SykgZrZd9T0DzWy4md1rZpPMbLSZ9a62GWtmPzezewGc3H6XIUMIoWH+oeIZ+H81Xv8tgC9U7Y+hMogGAPgAwNDq69cCOKZqXwHgiKo9G8BpdKyxAEbQ9rYArqz1+QBuBnB81T4RwD/p+Lej8tC4KYC5AFbp6OvX0f8AHAPgT1V7HCp/bQDAngBeB7Bh9Zo9BGBX6p8BAO4EcBwda3H1//0A/BGAVd97C4Dda3x2oHvkLAC/q9pTAOxRtc8GcMEyXk/uD/1bZp+vAWAygBkALqZr2p3aXAXgYLq+51ftAwHcWbW/A+DPVXtIdWyP4GMBWLH6/iHqK/Wl/jWrD2cD+GbV/jqAy6v2z7Hkd3Pt6vtWB7Aaqr9pqPzGTazae1bn4J0BTALQD8DKqMz3vaptjqL+Hwvg4o6+LvyvU3h6UPmRPMPMvodK/v071ddnhRAmV+1JqPx41uKazLH3B3Bbwb6dAIys2lcB2JX2XRtC+CiE8DSAmQDqimHp4hwN4B9V+x/V7SYeDiHMDSF8hMqgHED7/gXgLyGEK2scc7/qv0cBPILKda61RsVHWNLPVwPY1cy6AVg7hHBv9fW/Ati96PV6v6RYQghhMYDhAE4C8AqAa8zsBAB7mdl/zGwqgL0BDKK33Vj9n8fs7qj0G0IIU1B5KG3is2b2CCr3wCAAW7XJlyk56svOT6YPgdp9tR+A081sMioPKKtgyYPMZdU+vw5pP22Jyh+iB4cQngOwOYDBAO6oHueHqPyB20Tu97fd6fCYnlqY2WEA/q+6+aUQwkgz+w+ATwEYbWZfQuVB4z1624cgN6rjrczH7Qfg8DpPLRTYtbZLhZn1QGVCHGxmAZW/5IKZnVZt4vuK770HARxgZiND9c8DPjSAc0IIf2jmKZW6P9qTEMKHqEyYY6uT5FdQ+Qt/RAjheavE6q1Cb2m6F/x9sFSfmdlGAP4fgO1CCIvM7Ap3LNGKqC87PzX68Pjqrlp9ZQAODyE8xceo9vPLALZBxcP+Lu1+EZV+G4ZK7KMBmB5C2KnglHK/v+1OQ3p6QgijQghDq/8mmtlAADNDCBcBuAmVQdhS3gSwJgBU/+JfKVSCqZN9VcZhSWzKF1AJ0G3iSDNbwcw2BjAQQHLTlJAjUJEJ+4cQBoQQ+gKYhdQ7VsRZAF5FxR3rGQ3gRKvEC8HM+pjZujXarVA9BwD4PIAHQgivA1jUFGsA4FgA9xa9XrX9PSAymNnmtiTGDqjExzWNhQXVfjtiqTcuzX2ojDGY2WAsGeNroTJpvm5m6wE4oDXOWyyN+rLzU9CHuYrQowF8k+K0hlVf7wbgxapn/lhU/oht4jVUHBA/N7M9UblHelkliBpmtrKZsTewoWhIT08NjgJwjJm9D+AlVGIw1mrhsa4AcKmZvQPgfFRiSZq4GcD11WCubwL4FoA/m9mpqLgKv0htn0Llh3I9AF8NIfCTcBk5GsAv3Gs3oPIAUo978xRUrvUvQwhN3iGEEMaY2ZYAHqqOy8WoxA7Nd+9/C8AgM5uESvzQUdXXj0elv1dDxTv4xWW8fgWW3B87kZQqarMGgN+a2dqoxG48g4pr/TUAU1GJJZhQx3EuAfAXM5uCivz5MACEEB4zs0cBTEelnx5s1bMXjPqy81PUh0XJNj8BcAGAKdUHn9nVthcDuMHMjgRwD5y3JoTwspkdjEpoyImoPAxf1ORIqB5zeit+r1aj1MtQmNnlqAR0jW/m+64AcEsI4fo2OTEhhBBCtDqdxdPTJoQQvtTR5yCEEEKI9qHUnh4hhBBClIeGDGQWQgghhGht9NAjhBBCiFKghx4hhBBClAI99AghhBCiFDQre6tnz55hwIABbXQqohazZ8/GggULbNktm0dH9eVbb6XFOV999dVor7TSkttxxRVXTNoZrU/6wQfFC/V+7GNLFhR+++23C9/z/vvvR3vzzTdf1mm3GpMmTVoQQujV2sdtxLHJ1zzXn52VrjA2OZHlv//9b7LvnXeWlKhaffXVo73yyisv9+fyZ/HnAEC3bt2W+/gtoS3GZqOMy48++ijafL39tV9ttdWizWOU50sgvQdWXbXx1mXO9WWzHnoGDBiAiRMnts5ZiboYMWJEmxy3o/pywoS0ttmVVy5ZbqtHjx7RXnPNtCgyPxAtWLAg2v7Hs1+/ftGePHlytOfPT2sZvvLKK9G+55576jn1VsHMctVRW0wjjk1+oPU/ZNyfbYnPTuXtFVZYPkd3R49N/iHz3yW3j+GHj+eeey7ZN336ktpyO+ywQ7TXX3/9ZZ7bspgzZ8kwePzxx5N9+++/f7TrfTjm7wu0rG/bYmy25bhszndevHhxtLlf2QaAIUOWLHbw8Y9/PNovvvhi0m699daL9jbbbFP4uTze2vMPnVxflrpOj2h/xo4dm2xPmzYt2jwoZs2albTjQcsPPeuss07Sjn9c11577Wj37NkzaTd79uy6z1mk8EQ2evToZN+1114bbX6YfPnll5N27767pID5V7/61Wg/+uijSTue2J944olob7FFur7v5ZdfHm2euP1Ey9v+gaizeZ/4fOv9AfzKV76SbL/33pIl8fhHDkj77MILL6z5uUDqBRg2bFi0vReBH3T5Qcf/gXP77bdH+7XXXov2IYcckrQ7/PAlSya29KGvM5P7Xk89la6K9Oabb0Z7xowZ0Z4yZUrSjudPnlu5H4B0/PI4Gjp0aNKuEcdU17wbhBBCCCEceugRQgghRCnQQ48QQgghSoFiekS74rO3Ntpoo2gvXLgw2n379k3asUbP2VYck+DbcUxP9+7dk3b8Po7vaYRMi0aAA00/+9nPJvu4D19//fVkH8cZ8DXn7B9/fI7z8rFcDAcOc4wCAHzuc5+LNscbnHTSSUm7008/Pdo+3qCjgi5bSr1B2d///vejvWjRomTfBhtsEG2fvcVjkPvZB7Xytf/a174W7Z122ilpx8Gv/Lk+3o5jhDibiOPFgDTw+tvf/nayr4zLKz377LPRnjt3brKvf//+0eb+8/Mn9xHPhT77kpNOON7HB223VbD/8iBPjxBCCCFKgR56hBBCCFEKJG+JdoXTJYG0Xg6npXsZjLfXXXfdaOeKDrIE4t3d/L777rsv2pK3KpxwwgnR9pIIp7J62YplFpaIfGkBljW5BME+++yTtFtrrbWi/cYbb0R7jTXWSNoVSVO33npr0u6mm26K9rhx45J9nUHSYnJp2TNnzow2l4XwsjHLG/778zH79OlT8z1AKjNdd9110WZpCkhlLO7XDz/8sPBz2WZJDACmTp1aeAyWY3ifl2m6EiwzsUwFpOUINtxww2hfddVVSbtRo0ZF+8ADD4z2vvvum7Tbcssta36WLwXCZQsapYihPD1CCCGEKAV66BFCCCFEKZC8JdoVljKAVILKZQVxJhC7q71sxcdgd713ybO85eWbsnLZZZdFm6vx+uwavv65rCHuG792D6+Lxm5vL2tyv+VkCt5eZZVVot2rV7r8DktkN9xwQ7KPK/x2BnJLedx1113R5j7i6w6k1yq3ph2P0969eyf7WKK++eabo+2r87J8zbKHv4d4XSeW8PxY53vq/vvvT/btueeehe/rzPD1YAkTSK8vL8EDpLImS5XPPPNM0o7XLuRsvnnz5iXtWBpmeZMzyIBUSjv66KNrvt7eyNMjhBBCiFKghx4hhBBClAI99AghhBCiFJQmpodTKS+99NJk36BBg6LNKbOHHnpo259YyfCxOhwfwNo+r8IMpHE3HIfgKdLvffost/OfVVYuvvjiaPP18enADMdf+PcxuerHjI9T4c/meAPfjlNyOTbFrz7OsT8+XbezxfTk4Huar7WPmeJr6q8Vw9fNV27ma8+lBHLtOB7Hx/Tw+Ob5gittA+k9xWn5QBrTk4t96mxwHA/H0gDpHLfJJpsk+3g19e233z7a66+/ftKOU845TorfAwAPP/xwtDleaO+9907a8X3z4IMPRnuzzTZL2g0bNgzthTw9QgghhCgFeugRQgghRCnoOn6/ZTB+/Pho+8UKJ0yYEO3f/va30T755JOTdhdccEGzP9e7k3/6059Gm9OC//CHPyTtvGzQmeG0Y04ZBlJpkV3tXg7haqMvvPBCtDlNE0grvbK716ddcxVRv4CiSKUOL1Nwf+Zkw1w6O/dvURVnIJUmeJ9Pr+bzZXnEV4Hldr56LKfl+uq/nQ1OHeZr6EsHcOq4l415PHIf5aqb82f5dix1cDsvP/H9xZ/L5+qPz2nzXRmeB7kyvd/nx9F+++0XbZ4jucSAb8fSspetuM+4/3nRaCCt2M73np9zN91002j7auutjTw9QgghhCgFeugRQgghRCno9PJWvYvJceR4t27dkn0sd3HU/4UXXpi0O/bYY6M9fPjwws9iNyMfDwBeffXVaHN11OOPPz5pt8ceexQev7PBLs8111wz2ccVc9lF7SUVvlbsuvUu71122SXa7Br39wa78rtSxdbmcOKJJybbfC35ej///PNJO3aP++wPztDhPswtZlnvIpBFi0h6WJZ56aWXkn1cEdzfi/fee2+0uXpsZ8DLViwRsKTM1wZIpWK/GCmPEZYFc5Wb/bhlWLaqt885Y8tLJ3y+vjpxV4LHJV9fLwuylOTnRZ5b+Zr2798/acd9yxlbXMUZAKZPnx7tografjuXVTl37txob7HFFmhL5OkRQgghRCnQQ48QQgghSoEeeoQQQghRCjp9TI+PFWBYA541a1a0vWbIWjPHK/iqliNGjIj2EUccEe1+/fol7X79619He6ONNkr2cQwEa+09evQo+BadH66m7GMKOLaD4xJ8O47h4GqzPrWYq5QOGDAg2j51mfu5K5UHaA7f/OY3k+0xY8ZEm6+/jw/gfvIlGTjOgOM2cuOU9+UqN3M/cfwCkMafcBq9r9TL38V/1n333RftzhbT41OAOSaLx5gv8cBz5Oabb57s4zGXq9DNx+dYjXqrcPvxx2P1kUceibbvc74POY6yq8FxaEWlGYA0Vqd79+7JPv6N4zHgr9vll19e8xg+No7hucLHlvF8wPeon9+5fItieoQQQgghWgE99AghhBCiFHR6eStX9XXkyJHRXnvttaPt0+XYBccp5b7aLLt/b7vttmh7F/+WW24ZbU7hBdIF9NgFzSl7ADB48GB0Fdjt6l3UDLtGvRueKyqz25z7FUhdvlxx18uH3Oe5NNuujF/kj+9BXnzTpwoPHDgw2n7RQx4jPDa9K74o7Znd8EA6Bvk9/j5iqZjd8htuuGHSjvd9+9vfTvZtt912Nc+pM8AyEFB8T/OcAxRXUwaKFwX1c25Ouixql0tZL6rc7KUYDhXw45vHPsvcnRGeP9n2KwvwXOj7mfuMf5P8b9y//vWvaHO5FX8N+Xcsl4rOUhrLW0OHDk3a5eSz1kaeHiGEEEKUAj30CCGEEKIU6KFHCCGEEKWg08f05PjZz34WbV56wq/0XbQyMOunfh+XQPeaNpe39+m+rFezZs6rwAPA/vvvj64CXx+fOs6wHuyXCuE0dWadddZJtrn8Pq/c62NPuG/9cgQCuOGGGwr3ff7zn4+2X92aY3I4jsfHgRQtH+Pb8ZjLxZ/wfcWxSbfffnvBt+hacMqvh2M4fPwhl27IpRvz2PSp50Vp6rm4HU5T98fj8+Bz90tNcPyYP8bkyZOj3dljejh+huc3H9PD+3xKuI+Va8L/Pu27777R5t84347HNs+luc/l+CHfjo/h+7LemLF6kadHCCGEEKVADz1CCCGEKAWdUt5i9xe7vrjqMpCmwXF6o5et2I2bc7NxO3bP+/RQXw2z6Bjsyn/ooYcK39PZ4euYKzHA+7w71qewN+GrZj/22GPRZnnLp2ayy7jeFZ9FhaJxAKQyU65UQVF1Xt8XLJ3kJBY+j9wq4EXHBvKVoRudZ599NtlmiYilCF9+YLPNNou2H5tF1zF33fg9RX3sz8/fQyzT8D7fjj/Xn9NTTz1V+NmNjk8353AMloX87x2PMV/Ko+je9r9dLPUXjT2geLz5e4hlMa4s7dux7MplY4C0XElrIE+PEEIIIUqBHnqEEEIIUQo6hbzlI8c5op9ddWeffXbSrlevXtHmLAXvqsu5zRl26bF71mf/8D6fEcHfhd24Y8eOLfzczg73kc+6YdmJpRGfFVSU9cXueQB48MEHo81ufZY3gbQ6qHebizw++7GIogwtoHhxWT9eclk+DB8/V/WbyUmtnY158+Yl2ywt5ir18lzq5awiia/e8VLv9fVV61ly4exMf2/wvO3lb78Aa2fCX3e+t1kG8uPQX8ci6pWjcpm2fL15XPr5fcaMGdHmrErflzxmfXVmyVtCCCGEEC1ADz1CCCGEKAV66BFCCCFEKWjYmB7WCXPa4s033xztK664ItnH6cysf3rdsSgFPteO40W8lsq6eW4Fb9arn3nmmWTf6NGjlzrvroDXq1lf5mvq4wt8CmYTW221VeFnceqjjwfheK/Olp7c0XDasx+bRfECPo6u3nRo3ubYBh9XwrE/9cY2dCV8KrqPmWgiF1Pn4WvP1zsXW8X7/NzH/cdj3Zen4PGYi8/i7+irE/sYp86E7zvuo6Jq1UC60rxP+y4qK+DHG19vHtu+L3m85UpEcAwSz7m+4n7RSvJtgTw9QgghhCgFeugRQgghRCloNXmL3ZpFtofd315iyEkO55xzTrR/8pOfRHuLLbZI2rHbjd2zuRTJ3PkWLXjoXYTsxvWpukVSGrt7gSWVhX2KaWck5/IuWqzOp1IWLQq63XbbJdvcF9xfvh+KFsITy4Yrq3IpCCBNeWVXuZejihap9BTJn35c8HlwKYiy4Mt68JgrqooLpH1UbyVr31/8WdzPfk5juJ0f6zxH1LtIpZ9XOnMZCn9v83fha+8lTZ7Tcn2U++3ibT6+lxn5N5TP1193/ixORfcL5LI0J3lLCCGEEKIV0EOPEEIIIUpBq8lbrb1Y30033RTt0047LdnHi8lts8020c5Vl2SXt3fjcjt2x+Ukt1wmSU46KVqo1GfBNLkWO7Obtolc5gdnIyxatKiwXVGWVlFWF5DeDznXvbK3KhRJrx52gXsJgxdy5b7xbvQiGTnnHs/JpLydk1Xq/Y6dAZ/1xLBEwJLW0KFDk3bcR15yKKp8n5NEOKunKIMMSOc7Pzb5e6233nrR9hILf6/c4tB8Hnx+jYqXIPne5vGRk+VzFdB5XvSSIZMb55xVzMfz45JlK/6d9fcQH//5558vPKfWQJ4eIYQQQpQCPfQIIYQQohTooUcIIYQQpaDNKzL7ypB33nlntCdPnhztW265JWk3bdq0aPuVtDlNmbVKn7bJemUuFZ0pSkv3sL7stXXWU/0x+Jz4s7z+3dSus8cdAPk+4hV0eWVkf0379u1b89g+lb2oUmiurEBO1xZLUxRjAKSxJNwXuZRqPoYfBzx+uM98f/L90pVWT8/BMXAevqZF8RdAPu6G2+auab1za1GqtI8D4fHIFX19DAuv4O1jlfiY8+fPj3afPn3qOteOxPcJfxf+zn4MrL/++tHm308gjWnNpYQX9bOfI7kCNq8sMHHixKQdV17m+CwfP8b3kI9pam3KMTsIIYQQovTooUcIIYQQpaDF8tbYsWOT7bPPPjvanHLGrkUA2GCDDaK9ePHiaPt0xN122y3aXuJhdx/vy7ng+D2+HVdzZdeidx9ymmWuoiyngXr3f1ElUr4WALDTTjsBAP7+97+jK/HKK68k20UyoXd58+KxOdiNy8fzJQHYxVvGCr61qDedO7c4II8tlrf8/c3Hz5VlKJKb/efyPl+ptuhzOzuvvfZatP314PmJK+b2798/acdjxEvxfIychFVUMdjj06iL3sNjn9PmBw8enLTj3xk/p/M5sUTWGfBp9UVlTjgd3O/zVZ2L5jh/bfh685j1C1/z9ebfu1mzZiXtuNTI9ttvH+3bb789abf11ltH299rTz75ZLT9qgstQZ4eIYQQQpQCPfQIIYQQohQ0S956//33Y9T11772tWQfu7s4I4dtIHWhcmS3d0/mFjtj2AWby9DJwTITf5Z3u7KLkGUwzjry5+EXN2W3Y05+2X333QEUL7TZmeB+8Fk8c+fOjXYum81n8BXBLl92//vr2NoVxMsESyQsIQNpZVW+rr4/eV9RJheQzhe5CsR879S7cGZnJyfZF80zn/zkJ5N2U6ZMibaXVXgey1U35+Pze3xf8vv4eF6a4/Pg77jpppsm7a699tpoe/m0KAOsM+DnSJ4/+VrvuuuuSbui3zGgWEL2kiaPy9w44uPzPOv7iOFnAS/NcX/5+bi1s7nk6RFCCCFEKdBDjxBCCCFKgR56hBBCCFEKmhXT88orr+Diiy8GsHRKMcfn1FvxkVPFve7KOqbfx5ofa5K+miTHyfDxcumdXPXTf0dOkXzppZeizZUwAaB3797R9tolx5bwObEuCizRTLt6ddkivd2nLXbv3r2u42244YbRfuKJJ6LtVwlmvbozrLzcHhTFcPi+4HgRHxPA1zKXil6UAu3HHI8R7jMfr5eLOan3HDpbbFeuYjx/N27nYww51sqPsXpjeji+g9v5GCzft034OZKPwXOuj2HhVGkfM8bxlz7dutHx8Vn8XXgey8Vg5eDfP/7d9p/NsUX8Ww0AL7zwQs3PHThwYGG7Xr16RdvHYPG94avv52J6W0LX/kUVQgghhKiihx4hhBBClIJmyVtmFl2lXpZgWYjdbl5KYtclS0Q5V7OXJthFy8fz7r2itEgvGbEblt1x3i265557RvsnP/lJtEePHp204++Sq67JLr62XmStUfB9xFIJ31P+uvGidjnWXXfdaHMlTy8f8nZnWISwI/EyFd/ffizVKzPlFoNlivZ5aYfvna5Q5qEecjIjz5k8v+XkLZ6PgXTMsdThK17zmON9XqbhfuGFqJ977rmkHctWPEd6+ZHPlyv6Aun39yngjY7/LeSxwjKTr7LMY8DLvzyOihZl9tu5BX65HfeXlzS5Aj9LWFydGUjvZV++pbXHszw9QgghhCgFeugRQgghRClolrzVu3dvnHnmmQCWXjjy7rvvjja7HX10OLvJ2D3n3bMsR+UWwmPbtyuSvti16tt95zvfifYpp5yCerjqqquSbc7e8m5Bdi+za7kos6GrkXO7sovTZwt4V3kRnAnC7/H3Bl/vXBaMyGc7ermkKNvKU1S510sY3I6P5z+3JRV4O3v2Ft/DXnJ6/fXXo51b2Ji/c64yctGil0D6W8CS8o477pi0K5LBvHzKVb753H2WLG/7hSiffvrpwvNtdPwcydeH5SO/2sHEiRPrOj6PHX/teRzx+PChHiwf+nuK4d94ljE333zzpN19991X8/yApUMTlhd5eoQQQghRCvTQI4QQQohSoIceIYQQQpSCFgczXHTRRck2x6dccMEF0b7yyiuTdpwSvmjRomj7qoucpubjOTiljT/Xp8vxZ/F7fvjDHybtzjjjDCwPvFIxkGqXXp/luBWuUNm0en0TTTp0UeXazgTHCvg0S/5+nFq6wQYbtOizBgwYEG3W8n3ZA0YxPRWK7rXmrFJdtGK6j5cpSm3PrbLO5GIReIx1ZTiWIhdXwdf3P//5T7KP40Lmzp2b7ONrysf3fcJ9wcfzY52Pwe/xFZmnTZsWbU6bv+OOO5J2PN/7mCaOC/Fza2fGp3MzPMflUtG5//zvU1FMni8hwnM1jzcfw8uxmfxbzWnuQL56u4/xWV7k6RFCCCFEKdBDjxBCCCFKQYv9+j4Vm91fp556ak3bw2nujzzySLKPXZxz5sxJ9nEKG7v7vBvsG9/4RrRPP/30wvMoIlfhmfnFL36RbHN16tziceziGz58eM1jd7Y02lqwW9O7U1mCYne1d3/WC6fF8rXz15E/15+TSOH0Z6D+FHO2vXRWtMird8uzK54/N+cO94tPdlXmz58f7U022STZx3Mkp4D7tG+Wnv38yRIG95fvyyL5OjfWeZ8vT8FyKks2PvWcP+upp55K9vF909nnUJ4X+/XrF22fRv74449H21eoLpKd/XjjfdznPjyAJcOiFRL8Mfh75EIKcqsYtAby9AghhBCiFOihRwghhBClQA89QgghhCgFLY7pKYpvaQ577713TbtRqPc7Hn/88W18Jp0bjrEoiuUAUt2Z46Jy7bxez9pzTmvmOIJcOnuZqDdlPXf9i8ZMbiX1nGbPcRy5+6golqgrUxQPB6T3/oIFC6Lt+4tjIn2KOY+LXOkMjh/aaKONCtsVjW/fX1zKg+8nf365+CH+/p2tJAXHYAHA888/H+2hQ4dG28e6zp49O9rbbLNNso/HGF8Pf+35OnLZEL90E7fjvvRxRryPY9D8fcjn5Je4au2YS3l6hBBCCFEK9NAjhBBCiFLQufx+otPDFVY97ArNVR5ll6x3fXJ1V3aZetmF3auSt/J4eavelHAu15CTsDht1vcF93Wun7h/2S3f2VdSz8FV7L0kwpXJueSAlw64SrKXlLktX19fPZ9lJpbZOOXdw+fr2/FncX9xpXsglTi93MnzTE5ya0QGDx6cbPP5c8VjLzkdeuih0fZVyXkc8LzoxwfLgjx+fdkKXjGB5wc/H/M8zjKrLz/wmc98Jtr+Xs6FRLQEeXqEEEIIUQr00COEEEKIUiB5S7Q57CbnCH4gXaCQK7vmpIycvFVUAdTLGizR5BZrLBNF0o+/PuwSZ5c1AMybNy/a7Ir3WSJ8DJa3vAzJshjfO/54LAFwNXfOLALy8mpnY9CgQdH20hQvgvyzn/0s2j6TiSUSHotAKjs9/fTT0b7pppuSdiylcf/NmDEjacfXnvt8v/32S9px33L/+fNjyWXixInJPq7ovssuu6Az4StU++0m/CoGTG6RztwCwtx/LDP5eZaPwfO2p2iRWS9VckVxls7aAnl6hBBCCFEK9NAjhBBCiFKghx4hhBBClALF9Ig2h1f8Pfjgg5N9rO1379492nvttVfh8XKVsnkVadaJfWwHV33l2IgyU1S5dv/990+2R48eHW2uAgukMT6s9fu4II4X4PRV37cce8UxQn61cE6bHjhwYLRzMTydPX2dU5u/973vJfseeOCBaB9yyCHR5jTklnLmmWcu9zFaA47pOfnkk5N9u+66a7Q7W0XmHDxf+rgdjoP0cTZFJUB8OjiPNz6ev4Ycp8lzqY8X4ngkPoeiOCVg6Xi91lj9ITleqx5NCCGEEKJB0UOPEEIIIUqB5RaSW6qx2SsA5iyzoWhN+ocQei27WfNQX3YY6s+ug/qya9Hq/am+7DAK+7JZDz1CCCGEEJ0VyVtCCCGEKAV66BFCCCFEKWiIhx4zO8zMgpltUWf72WbWs8brzVpPoLntM8c5wcw2WHbLcmNmPcxscvXfS2b2Am0vfy6taFVa2l9mNsDMphXsO9vM9i3Yt9Q4MrOjzewHZranme28fN9ItJRqH0w3synV/t8hMw8fYmanFxxH/djBmNn6ZvYPM3vWzB43s1vNbLNmHmNtM/t6W51jW9IoBQyOBvAAgM8B+FHHnkqLOAHANADzltGu1IQQXgUwFADM7EcAFocQftW038xWCiF8UPvdrY+ZrRhC+HDZLcvJsvqrhcc8q9brZrYiao+j/QFcBOBgAIsBjFuezxfNx8x2AnAQgG1DCO9VH3QKH3pDCDcBuMm/bmYrAdgT6scOwyrFqUYB+GsI4XPV14YCWA/AjMxbPWsD+DqAi1v5FNucDvf0mNkaAHYB8D+oPPQ0vb6nmY01s+vN7Ekz+5u5amJmtqqZ3W5mX65x3FPNbEL1L5MfZz7/fDN7xMzuMrNe1deGmtn46ntHmdk6Ra+b2REARgD4W/UvoNpVoERNzOwKM/u1md0D4NzMtR9rZiOqdk8zm121B5nZw9VrP8XMNq2+fgy9/ofqjyrMbHHV2/AfADt1yJfuQhRdfwArmtllVe/AmKZxUe3vI6r2bDM7y8weQOUPn2QcVcf7UAALAXwVwLer+3Yzs/7VMTul+n8/Ov6lZna/mc0ws4Pa+ZJ0RXoDWBBCeA8AQggLQghND6bfrM6fU63qqa967H5XtXl8XwPXjx3wXcrOXgDeDyFc2vRCCGEygAfM7Dwzm1bty6OAyu9zdXw19fGh1bf9AsDG1X48r92/xXLQ4Q89AD4N4PYQwgwAC81sW9o3DMApALYCMBCVh6Mm1gBwM4CRIYTL+IBmth+ATQFsj8qkOdzMdq/x2asDeCSEsC2AewH8X/X1KwF8L4QwBMDU3OshhOsBTATwhRDC0BDCOxDNZTMA+4YQvovia1/EVwFcGEIYisqP5lwz2xLAUQB2qb7+IYAvVNuvDmBaCGGHEMIDNY4nmsdS17/6+qYAfh9CGATgNQCHF7z/3RDCriGEq7H0OBoG4LEQwiwAlwL4TXXf/QB+B+DK6n3yN1S8QU0MALAHgE8BuNTMVoFYHsYA6Ft9iLzYzPagfQuq8+clAP5fwfubxvfhWLofRfsyGMCkGq9/BpXfym0A7AvgPDPrDeBdAIdV+3gvAOdX/xg5HcCz1X48tV3OvJVohIeeowH8o2r/o7rdxMMhhLkhhI8ATEZlMmviXwD+EkK4ssYx96v+exTAIwC2QGUS9nyEyl8fAHA1gF3NrBuAtUMI91Zf/yuA3Yter/dLiizXhRA+bOE1fgjAGWb2PVRqM7wDYB8AwwFMMLPJ1e2mtQk+BHBDa3+BElPr+gPArOpfkEBlkh1Q8P5rCl4HKtLWbQX7dgIwsmpfBWBX2ndtCOGjEMLTAGaiMv5FCwkhLEZlPJ0E4BUA15jZCdXdN1b/z/XxdZKRG55dAfw9hPBhCOFlVJwA2wEwAD83sykA7gTQBxUprNPSoTE9ZtYDwN4ABptZALAigGBmp1WbvEfNP0R6vg8COMDMRoaliw0ZgHNCCH9o5impaFHH8Naym+ADLHlIj3+5hxBGVqWqTwEYbWZfQqX//xpC+H6N47yrCbjlmNlhWOJ9+1LB9Z+Jpcdukeyb6/v9UOwh8oQCu9a2aCbVMTMWwFgzmwrg+Oqupn728zNTz/gW7cN0AEfUeL1oIbovAOgFYHgI4f1qWEGn9px2tKfnCFRc1P1DCANCCH0BzEL6V1sRZwF4FbUDqUYDONEq8UIwsz5mtm6NditgyQ3weQAPhBBeB7CI9OZjAdxb9HrVfhPAmnWcs8iwjGs8G5W/NgEatGY2EMDMEMJFqARPDgFwF4AjmvrczLqbWf+2/wZdnxDCqKpLe2gIYWLB9W8pcRxVvX4rVYOpk31VxmFJDOAXUEmEaOJIM1vBzDZGxcP31HKcU+kxs80pVguoyCAtrTKsubJjuRvAx43iYM1sOwCLABxlZitaJbZ1dwAPA+gGYH71gWcvAE3zaKftx45+6DkalUhy5gZUHkDq4RQAq5jZL/nFEMIYVFzfD1X/KrketTvoLQCDzGwSKh6ns6uvH4+KpjkFlQG+rNevQCV2QIHMy0/RNf4VgK+Z2TgAnCZ7FIBpVRlrC1Qeoh8H8EMAY6rHuQOVYEzR+ix1/ZfjWFegOo4AHIKKO72JmwEcRgGw3wLwxWr/HguAl9l+CpWH5dsAfDWEkC45LZrLGgD+apX05imoxFj+qIXH8v0o2pGqKnIYgE9YJWV9Oip9ORLAFACPofJgdFoI4SVU4uVGmNlEVP64eLJ6nFcBPFgNfO5UgcxahkII0XCY2eUALg8hjG/m+64AcEs1wUAIIRIapU6PEEJEQghf6uhzEEJ0PeTpEUIIIUQp6OiYHiGEEEKIdkEPPUIIIYQoBXroEUIIIUQp0EOPEEIIIUpBs7K3evbsGQYMGNBGp1LMm2++mWy/996SYq89e/b0zVuNV155JdleddUlJXjWWGONNvtcZvbs2ViwYEFRtcwW0559+dFHH0V7hRUa4zmbA/jNWv3yFjJp0qQFIYRerX3cjhqb9fL+++8n26+99lq0P/xwSYFsn1ix5ppLymu115irl64wNsUS2mJsNkpfLly4MNpvvPFGtD/44IOkHY8/HpcrrZQ+KvBYXH/99VvtPFuLXF8266FnwIABmDhx4nKdTEt+bO65555ke+bMmdH+n//5n+U6nxwXX5wWex4yZEmx2V13rado9PIzYsSINjlua/RlvbzzzpI1WPnBsSPhwe4HdFtiZi2tZJulLfuzORmeRWP6hRdeSLZvueWWaC9atCja/uFor732inZuzBXNK/7cW/MBtyuMTbGEthibjdKXI0eOjPZdd90V7QULFiTtePzxw5F3Luyyy5K1v089tfHWG831ZWP82S2EEEII0cY0THFC/msPAA4//PDCfSuvvHK0p0yZEm12xwGplMISC7v6PC+99FK058+fX3i8VVZZsubaww8/XHg8kXp3/vvf/yb7+Hr36dMn2jnvAnuO3n333cJ9r776arS7d++etOvfX0txtQY5zwl7c/74xz8m+7g/evVa4oXmcQqk3tYZM2ZE+8QTT6z7PJiOkjWFaA3qDRVYZ511ku3XX3892t26dYu2l6beemvJ2rCrr756tJ999tmk3ZgxY6J95plnRtvPx0yjjD15eoQQQghRCvTQI4QQQohSoIceIYQQQpSCdo/pKdLyvv3tbyfbTz75ZLQ33XTTZN+KK64Y7QkTJkS7b9++STtOdT/ggAOi/dBDDyXtOOZk8eLF0eZ0Wf+5Tz/9dLSvuOKKpN0JJ5wAUZuvfOUryfbtt98e7bXXXjvaPqbn4x//eLQ5w8DHgPD9xf3v282bN68ZZ11u/Jjla+n3jRo1KtpXXnlltH1WFscjcBxBjx49knYbb7xxtO++++5oDx8+PGm3zTbb1Dy/RimRIERrkLufn3nmmWj7+Y7HC5eLWG+99QqPzzGyHMMKpDGRs2fPjvb3v//9pN0555wTbZ4r/Pm15zjVjCCEEEKIUqCHHiGEEEKUgg5NWWcX11NPPZXsY/eZr4zMKa7sguOUViBNuRs7dmxhu6LidN7lxunWvXv3jja78ADJWzmmTZuWbBdV8+Sq2wDw4osvRpslSJ96vtZaa0WbXbKNUhSxM+KlxpwrmtPUuWQA9x8AbLTRRtHmNNd77703acdlDFiSvOiii5J2l1xySbQ/9rGPRbsj3ejLQ9M1b8/U3lwhx1y6Mc/BfH19u5YUkGyUNOf2pN6CmrNmzUq2OXWc50EgLQ7KhVm5xAeQ/sa9/fbb0fahI3wMTo+/7bbbknacHn/66adH24/D9pSkO8cMIIQQQgixnOihRwghhBCloEPlre9973vR9nIGu6g5cwdIs6hYtvCuOl47hCUR7z7k7dVWWy3avsIzu+H5HFhGA4Abbrgh2lxZWqQVmIG0Mi9fRy97sXt24MCB0fayFd83bD/44IMtPGPRHFlhiy22iDZXTvfjoKi6Oa+1BaTudq7M7mVSrjibq/DcWeStoms+derUaPP15fkNaNm6YLl+zu3jubAlx2/p53ZVct+ZK5HfcccdyT5eH8uvlfXyyy9Hm8M5/IKjLCfzGpf+/uLfQp63/aLAXIl9/Pjx0f7nP/+ZtCtaPcHvaw06xwwghBBCCLGc6KFHCCGEEKVADz1CCCGEKAXtHtPDeh1XRmZNHkh1eR/Tw3A8jo+t8fEjtc4BADbYYIOax/MxQvw+1jR9u9///vfRVkxPil9lneMBOK6L43GAtHIov8dr0kWxIl4nnzNnTrS14nrr8cQTT0R74cKF0d5kk02SdtOnT482xwH52D5Om+Ux56ulc/xeLqanM6RAf/TRR/F7X3vttcm+m266KdpDhgyJto97uO+++6Ldr1+/aHM1XiC9br7yPZcK4Wvq4WPyXO3PiWMk+dhciR1I+yw393P/+XmF5wW+p3z5E46RaVTuueeeaD/wwAPR9v3F143jvYD0t5HnVj8GuIr9LrvsUvN1AJg7d260OUbIj0uet3lu+MlPfpK043R7pawLIYQQQrQCeugRQgghRClod3mLXVfsqjvuuOOSdryQaM79yS5TX1mZ06E53ZWrKfv38eKH3s3G7nU+nk+z9S7pssPXbf78+ck+dr2zbOUXqGT3LKepe/e3T61swi9kydV9JW9VYOmH7Zy7+U9/+lOyveGGG0Z70KBB0fYyE49Bdp17uZJd+1tttVXhOXEK7He/+91oe5k0t1hqo/D666/j5ptvBgBMnjw52ffTn/402vfff3+0eeFeIJV2hw4dGm1fxZdlEL8QM6c9c8rzggULknZc5oNlMF40GkjHILfjNHwgHd889/uxzhIeV/8G0u/M8inP70C6cHSjctVVV0Wbf6u8pMf4e5uvHc+z/pry7ynfG74swRe/+MVoP//889H2qx2wPM2Vm1nqam/k6RFCCCFEKdBDjxBCCCFKQYdWZGauvPLKZJuznu66665kH7suOXMqt4gZu1a9648lEZZivFzGmQ7f//73o/2d73wHohjO4vHXlF2ePkOAKcriYDc+kPYRf5av8OyzBUU6LooWkQSAu+++O9qTJk1K9rE0wdffH4MXROS+YEkaAA4++OCa+zh7xG+ffPLJ0b7wwguTdnwe9S7s2N6svPLKMaPUywoTJ06M9sMPPxxtXtjRb7MMtMceeyTtuNK5n4P333//aM+ePTva/pyOOuqoaLN8zdIGkM4DvM9LHTvvvHO0ed720gmHGPh5he8vzthiSRBIZZpGhaV+Hpd+Dtt4442jnZtLGS8n8zZ/lh8bLF3ye1gGBdKwBJbLWBJrb+TpEUIIIUQp0EOPEEIIIUqBHnqEEEIIUQo6NKaHY2685s8rlbOeDADbbbddtFnH9NVcWbNnfTJXpZV5/PHHk23WSTlNU+RhLd+viu5T05vwK9wzuaq6vI8/y1fr9mm3IiW3cva4ceOi7ctJcOwVx4sMHjw4affUU0/V3OdLDnAcAKdQ+9RrToHnuC6+94A0LsjPA/WuFt7WvPvuu/H68DUE0lgIvm7PPvts0o7nzClTpkTbl9fgqvW+ajangfPq2VxmwsMlAvr27Zvs4/mUv5evaM9wRd+mNP5a+/z99cwzz0Sby5/4WJfcZzcKPFfx76SPn+GVBXwMJMfd8H3uf/uKfid96Qe+D3mfr8jMldc333zzaPvrzqUDfKXp1kaeHiGEEEKUAj30CCGEEKIUtLu8VVTp1csZ7IJjtzaQusCLqsgCxdVXvVubP5uP4dtJ0mp9uESAXySPYemSXbW+T7j/cguT5qqZlpV6F+Nk+YhtD0siLEUAwHPPPRdtTl/2n8uufU5R9nI4nwf3ra9ovPfee0e7UeWtlVZaKcpwvoI5l15gSct/F35f0XuAtJL1iBEjkn0sYWyzzTbR5pIFQCo1br311tFmWQlIU9HHjh0bbS+RPvLII9HmPvG/ESzh+YVEWT7h4/vfiCJ5vZEoSj/3cxhLlf43kyWoXOgAhwQUpa/747HtZSue33ls8+tAKndK3hJCCCGEaAX00COEEEKIUqCHHiGEEEKUgnaP6SmKFcjFEBQtQQCkmqxPWeclCorS13PH86XNi2jUcvaNAmvPPhaDrzHHgHjNl3V5Tn3kUvxAWn6e+8F/bqPEbzQSHBfC18fHS3AMzoABA5J9rM1vtNFG0fbxHdw3L774YrQ5JgRI40p4SQIfo8WpsRzD4lfw5pieRh2nH374YVwNnK8hAOy2227R5pXVfSzFlltuGW0eEz7N+ZRTTom2j9XheCpeCmiXXXYpPCfu/wMPPDBp99hjj0Wbl544+uijk3ZFy19wXBEAjB8/Ptq+NAGz1VZbRZtXXAeWjjVrRLi8A69O73/vGP+bxG35N86PAZ4nc3GPPP6K4ij98YtKwwDpON1zzz0L27UG8vQIIYQQohTooUcIIYQQpaBhVlnPuZp9KjOnyLGbLZfyzK4672ZjiYVd/EpRbx24xICv7MnkUsxZ4uQ+8is5swzG94OXt3ISZ1kpcj/fdNNNyTa72FlqBNKxxC51lhiANKWa7w8vU/AYZLnap/E2yUFAKudwGq+nXvm6vfnggw+iDMWSHpCm4HOavp/7eAVuvgYsMQHAPvvsU3gMllV+9atfRdvPi1dddVW0Wd7yK5izbHHPPfdE299DLNVdf/310X7ttdeSdlxB2svh8+bNq3k8fx/Wuxp5e+LHAI8Prrrs5S2e03g8AOn14fHhrxsfg+dMPx8zLJd5SYyPwb/x/vd+0qRJhcdvbeTpEUIIIUQp0EOPEEIIIUpBh/p3660A62F3KLtxvduVXXIsieSqP/O+bt261X1Oohh2oXpJgd2fOXmLK4yyi9dTVGHVf66XxUTxGPTZWzxuubIukPZn//79o+2lCZZceJFCn23FciWfn5cAeKzy4rJ+AVOWBHJZoR3JaquthuHDhwNIKyYDqaTDi6zee++9STuWDzlDy2dvnXvuudH21+O8886LNmfEXXjhhUk7zvJi+fqhhx5K2h188MHR/ta3vhVtfw/xvcEZW14G4wVIOcsPSBcgZcnFy3s77rgjGg2uVg4Uryzg4bnPS5U8t+ZkXR6/udUJit7j4c/KZW/579yWyNMjhBBCiFKghx4hhBBClAI99AghhBCiFHToKustrYjKaYasVXrNkPVl1vY5hgAoXrXba5W8yvM666xT+LmNWum1o6h3RXPWoXN9ydeeVwVui3MqE0VVqqdNm5Zsb7vtttH2cSAzZsyINvfZhhtumLTjMcJxG1yV29O3b99oz507N9nHcWP8PfwYfvrpp6PNcR+NxAorrBDjkm677bZk36BBg6LNlYxfffXVpB1v83UbOXJk0o7T3ufMmZPs43iXjTfeONrHHnts0u7GG2+MNsd+8H0CpKuxc2wVz6tAem/w9xg2bFjSjvf5YxxwwAHR/stf/hJtn6KdizPpKHzcFc+LuQrHuZRwHgcct+rjW4uuhz8eX0c+P56bgTQ+i0sH+OPlSpm0NvL0CCGEEKIU6KFHCCGEEKWgYRYc9Slx7I7705/+lOxjlxyntPpF9/gYbPuUPU71Y3nLV3P9/ve/H+1LL7205rHF0nB/5RbJ43vDy0/sQmVJxae282exzOFT2XPnIVK5wEtO7H73KeYsVXGa88yZM5N27Ebn8gF+AUhOl2d5xKeic78/+eST0fZjkxc+bVR56913343VkL1ExN/n8ccfjzYv+gmk9/uDDz4Y7SFDhiTtuDovLwIKAP369Yv21VdfHW2u1AykqejcLw888EDSjsfw0KFDo+0laq74zfPxv//976TdZpttFu1vf/vbyT6WWfne8L8/XiZtBHyJiFw1ZKZIBgOK50U/PuoNzeDfUD62LxvDMlgutIVLz7Q1+rUWQgghRCnQQ48QQgghSkHDrLiXc6vdddddyXZRBWUPu9Y4OtxLHSytsc2VXYH2XRStK8F95GVMdnmyq9XLT5wVwLJJTgbLZWYUVW4WFfi6coYPAOy3337R5sq/QNpvnLHFMjSQSmTPPPNMtH12DVf75QrPXsrm+YMXlfRZTbkFSBuFVVZZBZtuuimApb8n3/tcoZgX/QTSa7DllltG+6c//WnSbqeddoq2vza33nprtFly8dWPWdLiRWH/9re/Je0OPfTQmp/lq/Gy5Pbiiy9G+5BDDkna8b02atSoZN8OO+wQ7abq1sDSFa5ZImsUfCYa9znjM6W4Xb1Zan4+5t/W3G8y7+Nj+Hl7++23jzZXUffztq/Y3pbI0yOEEEKIUqCHHiGEEEKUAj30CCGEEKIUdIqYHl+hkttyvIhPRWcdkzVEX0WWj5fTNP3KtUWwxql09hR/Dfka87XyKcl9+vSJNq807bVhPsZbb71VeB71poGWlRtuuCHaPmWdr7m/xv/5z3+izdWEfTuOC+FSENdcc03SjtOZOabOp7juu+++0eaK7S+88ELSjuOCGpUQQow586noHKtxzz33RHvixIlJuw022CDaHGczcODApJ1PP2d4bO69997R9jFeHO/Dc+vWW2+dtOP4Do5V8nEgHMfF8ztXlgbS6to+pofP6bDDDou2jwvy6eGNgI/j4uvDfdKtW7ekHaf6+37lVHL+ffKxPkUxlrkKz/yb6c+9KTYNSO8bH3PUnvOxfpGFEEIIUQr00COEEEKIUtCh8la9i49y2iKQyljsJvMp5kWVOL3kxOdRVLkSSN1zkrDqp8g9C6R9yWUFvLuT3fXrrrtutL1swvIZ95+X1ZSynoerJHt5ixcg7d27d7Lv0UcfjTb3ta/UypILp976fmJ3OY9N75bntHeu6uwlFpZEGpX3338/znmcvg2kcw2XAfDfk9935ZVXRtuHCnTv3j3avjIyV3LmscTp4ECa9s399c1vfjNpx/JkbiFRlpxmz54d7bvvvjtpx4uK+srVnALNc7WXyBpxwVEeG0B63/O8uMUWWyTtevToEW0fHsBSWK5CddHvmv+NK5K+/LzK8wNXQ/elZnLHqDespF70ay2EEEKIUqCHHiGEEEKUgk4hb3kJo8hV57O3ij7Lw5+dOw92+XP2iK+MKVJY3splC3Bf+uycNddcM9osb3lXaNE95eUy7kuxNHx9fIYcS8q8uCeQyiC5McdjldvlKnbnxiZn/LCE4TONvNu/EVlxxRWjPOUXxORKxiNGjIg2y78A8Oyzz9bcN2DAgKQdy0c+q3WvvfaKNt8DXlbhSrssl3kpjY/BUsycOXOSdnwMlip91V6W37g6NQAceOCB0ebFR/k+AYBPfepTaDT8fc5zHO/zVc6LqiQD6XjLhWbkVjhgihbw9r/V3M98f3GGJZBKevPmzUv2tXbGpTw9QgghhCgFeugRQgghRCnQQ48QQgghSkHDVGTOwdV4gVQPZD3Ra6EcD8C2j+/g9+ViCFhbZR1bMT15+Jr6GJyiSpw+9sLHIjThU3o53qSoCilQv3ZdVlhX33nnnZN9nEI6derUZB/3b25sMkXjFEj7jW1fToI/l9OhOU0aSGMOfPyBL3nRkTTFTPhqxQ899FC0Of3e398c/8IVif04GjduXLR92jtv83lcdtllSTu+H3r27BltP4b333//aHM80rnnnpu0mz59erS//OUvR3ubbbZJ2p1zzjnR9mVN+DeC46K4QjCwdMxXI+BjU7lved7y5SJ4Ls2VBuGx4sdR0efmUtbZ9hWZ+bdxyy23jDZXawfScgl+lXnF9AghhBBCtAA99AghhBCiFDRMyrqH3XjeZVaUiuxdermU5Xo+17v++HzZnbrxxhvXdWyxtKzE/cIudO/i9QslNsHprUDqUvcpnSIPlwng6+jHKadD+xTglpCTtxh2t/sqrSxT8HzBC5ECwJgxY6Lt5ZdGkbdWXnnlmKrtqySzRMDjxadzc8r2HnvsEW2umA0AO+20U7T9GOOyBfxZXiLj1HS+pl6a40rLXNV70KBBSTtOc+Zjz5o1K2nH866X9/h+4N8BX12cP6tR4Mr0QHr+fE192AfLnf4YRRWUvWxV9Fm5xbf5GLlKy3zf+DAHPoYvV9LayNMjhBBCiFKghx4hhBBClIIOlbdyGR2chZOr4stuzXoXj8u1433e9cef5SU3UQy7Qr3MWFSl08tbRdKDl7DYvc6u1pw7VVRg+YFd50899VTSjvvQZ5BwhWaunO4pqoJeb5aIz7ziSsV8Dr169Urascv+8ccfT/Zx9d+O5N13343X/B//+Eeyj6src5VyzpoCgJEjR0ab5UifocWSka/+vN9++0WbZTHOjgOWloya8Fk4vCgsy0qcrQWkY53bTZ48OWk3ZcqUaPssTr4/eC7xC86OHz++5rl3JH7u4/HBVa394ql8fbwsyr9dud/d3HkwPLfy/O4/11dernU+ntaQzHNo5hdCCCFEKdBDjxBCCCFKgR56hBBCCFEKGrYic66aa1FaeS72h8lVZM5pnxxTwKvCijxcGdn3CafF8vXmeAWguHJoLqaEdX3/uTm9uqxwrMbzzz8fbZ/KzFVtR40alezjGC0ep7k4Am7ntX5+H6dl+zIRfE587/gYA44/qDcGsL1ZYYUV4nfguBogjXXktG+/QvoOO+xQcx+PNyBN7fZlALiaNcfO5Vaq52vvU9F53vUVlBlOU+dV4H06dL9+/aLt44w4ZZtTpX26vV+dvRHwqf4MXwPf57wvN7/xXOp/C3lMcLvcageMH29Fx8vFdubur9ZAnh4hhBBClAI99AghhBCiFDSsj5/dXd5Vxy7eetPvmHrfk3N/+xTJet9XdjbaaKNkm1PJuQxAUQVmj69Kyumv3M/+HpI8uTScss5yBssNQNpP3p2dq+TM5FJWGXaJ83tOOOGEpN1BBx0U7U984hPRZgnEU2+V9vbmo48+irKTT7nn8XLnnXdGe9iwYUm77bffPtqczn7//fcn7bisgJe+OOWcFy31i7g+99xz0eYQAE6vB1Lpi+VTL9Pwd+T70Kc/szTlyyPwgpb77LNPtDnlG0jls0bBl2Ng2ZH3cZkGoP6K4vVWQC8qK5E7hpdI+R7isez7nOVI/n1vC+TpEUIIIUQp0EOPEEIIIUqBHnqEEEIIUQoaNqaH8fofr8LakuUEvI7JWiOn/fkUSf4sX/adaUmcUVeGS9371FJeJZ1Tknfeeee6ju1jNrjPWBv28QCNqOV3NBwXwdfVa+zcT/661ru8xLrrrhvtefPmRTu3rAiPud/85jdJux/84AfR3mabbaK9ySabJO04DqatV3NuKaussgq22morAEvHd3Bs2pFHHhltP1fxEhtc1sGXeOBrdcsttyT7OJ6I47p8POPgwYOjzctG+KVf+D7iWDx/TvxZPDf7e4Pjgvh+AtLV6Hl5Db9S+1FHHYVGw/8+cSwUx0/5PueYHr80CI+/ovIfQBo3V7Qye63tJnw/cEkE7pN6V5JvC+TpEUIIIUQp0EOPEEIIIUpBp5C32P3tyVX7LaLeND3vkmfXMn9uc45fRji11Kesr7/++tGeOXNmtIcOHVrXsYcMGZJsr7POOtFmuca7gj/5yU/Wdfwywano7Jb2q2WzLOTlRXa/swzmrz+nDi9cuDDaXv7kz+bx593jRenLfoV4Tm2vN8W3vVl11VXjauh+VfS25Ljjjmu3zxL1w/IWy0++KvmYMWOi7aVbDhHhUg1+XDL1hmnkKi3znL7HHntE25cQ4ff5sgKtjTw9QgghhCgFeugRQgghRCnoUHmrXvcZZwQAS1eibMIvVMbbHBHuo8OLFmfz1WZzrkBG2VspLCmw3RqwyxQAxo4dG+1cloJYGnaBc9VdzrADgA033DDaI0eOLDzeY489Fm0vUbOMxQtTHnzwwUk7HnO5xSw5S4vf85nPfCZpx+cxfPjwwnMXoqPwVY3nzJkTbZa3fKgAS/a+8jb/lvExfGX0ogVCc1nSvM/LapyFy4sC+4xQlrgXLFhQ+FmtgTw9QgghhCgFeugRQgghRCnQQ48QQgghSkGniOnxK2lzFVhOHfexB5zWypVNvWbKOibrk5xyC6Q6ZG6VdZHCKYg+1bhe+NpzDJaPxyqK4/HxWJwi6St+lxWOj7rgggui7cfLeeedV9fxuNov2zn8auEtge8BP3fwHMGrsQvRKPi4R64izjE4vvrx1772tZp2I3LIIYck2zw/H3744W362fL0CCGEEKIU6KFHCCGEEKXAmlM92MxeATBnmQ1Fa9I/hNBr2c2ah/qyw1B/dh3Ul12LVu9P9WWHUdiXzXroEUIIIYTorEjeEkIIIUQp0EOPEEIIIUpBp3voMbMPzWyymU03s8fM7Dtm1um+Rxkxsx7VvptsZi+Z2Qu03bJcdtGwmNn6ZvYPM3vWzB43s1vNbLNmHmNtM/t6W52jqB+aex8zs0fMbOdlv0s0GmUfl50upsfMFocQ1qja6wIYCeDBEML/uXYrhRA+qHUM0fGY2Y8ALA4h/Ipea9c+M7MVQwj1LagmmoVVinCNA/DXEMKl1deGAlgzhHB/7r3uOAMA3BJCGNwW5ynqx829nwRwRghhj2W8TTQQGped0NPDhBDmAzgJwDeswglmdp2Z3QxgjJmtbmZ/NrMJZvaomR0KAGY2yMwerv7VMsXMNq22/Xf1r5hpZnZUh365kmBmV5jZr83sHgDnmtlQMxtf7ZdRZrZOtd1YMxtRtXua2eyqvVRfVl8/hl7/g5mtWH19sZmdbWb/AbBTh3zpcrAXgPebJlYACCFMBvCAmZ1XHWNTm8aZma1hZndVPQhTm8YqgF8A2Ljaj/VVRRTtwVoAFgHZvoOZnWlmT5rZHWb2dzP7fx12xgLQuOzYisytQQhhZlXeaipPuROAISGEhWb2cwB3hxBONLO1ATxsZncC+CqAC0MIf6vKKisCOBDAvBDCpwDAzLq1+5cpL5sB2DeE8KGZTQHwzRDCvWZ2NoD/A3BK5r1L9aWZbQngKAC7hBDeN7OLAXwBwJUAVgcwLYRwVlt+IYHBACbVeP0zAIYC2AZATwATzOw+AK8AOCyE8IaZ9QQw3sxuAnA6gMEhhKHtctYix6pmNhnAKgB6A9i7+vq7qN13wwEcDmAYKr81j6D2PSHaj9KPy07/0FOF17O4I4TQtE79fgAOob8uVgHQD8BDAH5gZhsCuDGE8LSZTQXwKzM7FxW3Xd2uPrHcXFd94OkGYO0Qwr3V1/8K4LplvLdWX+6DyoQ7oeLNxaoA5lfbfwjghlb/BqJedgXw96qs+LKZ3QtgOwC3Afi5me0O4CMAfQCs13GnKWrwTtOPnJntBOBKMxuMyvxbq+92BfCvEMI71ffc3CFnLeqhNOOy0z/0mNlAVH7Imn7U3uLdAA4PITzl3vZEVd74FIDRZvalEMLdZjYcFY/POWY2JoRwdlufvwCQ9lkRH2CJHLtK04shhJG+L1Hp97+GEL5f4zjvKo6nXZgO4IgarxctuPcFAL0ADK9652aD+lk0FiGEh6p/+fdCZc6s1Xf1La4o2pPSj8tOHdNjZr0AXArgd6F2RPZoAN+06p/7Zjas+v9AADNDCBcBuAnAEDPbAMDbIYSrAfwKwLbt8R3EEkIIrwNYZGa7VV86FkCT12c2Kt4bgAZtrb4EcBeAI6wS6A4z625m/dv+GwjibgAfN7MvN71gZtuhEgdylJmtWB2/uwN4GEA3APOrE+teAJr6600Aa7bvqYtlYWZboBIW8CqK++4BAAeb2SpmtgYqf5iIjqX047IzenqadOWVUfnr/yoAvy5o+xMAFwCYUn3wmQ3gIFTiPY4xs/cBvATgbFRceeeZ2UcA3gfQ2MvUdl2OB3Cpma0GYCaAL1Zf/xWAa83sWFQGbhNL9WU1nuuHqASzr4BKf/4vVA6+3QghBDM7DMAFZnY6KnEfs1GJz1oDwGMAAoDTQggvmdnfANxsZhMBTAbwZPU4r5rZg2Y2DcBtIYRT2/3LiCaa5l6g4hk4vipLF/XdhGr8x2OojL2JAF5v97MWEY3LTpiyLoQQonNgZmuEEBZX/4i5D8BJIYRHOvq8RHnpjJ4eIYQQnYM/mtlWqMSB/FUPPKKjkadHCCGEEKWgUwcyCyGEEELUix56hBBCCFEK9NAjhBBCiFKghx4hhBBClAI99AghhBCiFOihRwghhBCl4P8DZojOzUdubqYAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.figure(figsize=(10,10))\n", + "for i in range(25):\n", + " plt.subplot(5,5,i+1)\n", + " plt.xticks([])\n", + " plt.yticks([])\n", + " plt.grid(False)\n", + " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", + " plt.xlabel(class_names[train_labels[i]])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "coordinate-jefferson", + "metadata": {}, + "outputs": [], + "source": [ + "model = tf.keras.Sequential([\n", + " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", + " tf.keras.layers.Dense(128, activation='relu'),\n", + " tf.keras.layers.Dense(10)\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "august-opinion", + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "pending-madison", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "1875/1875 [==============================] - 13s 6ms/step - loss: 0.6163 - accuracy: 0.7844\n", + "Epoch 2/10\n", + "1875/1875 [==============================] - 11s 6ms/step - loss: 0.3804 - accuracy: 0.8625\n", + "Epoch 3/10\n", + "1507/1875 [=======================>......] - ETA: 2s - loss: 0.3322 - accuracy: 0.8772" + ] + } + ], + "source": [ + "model.fit(train_images, train_labels, epochs=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "personal-deployment", + "metadata": {}, + "outputs": [], + "source": [ + "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", + "\n", + "print('\\nTest accuracy:', test_acc)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "three-lighting", + "metadata": {}, + "outputs": [], + "source": [ + "probability_model = tf.keras.Sequential([model, \n", + " tf.keras.layers.Softmax()])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "plain-lease", + "metadata": {}, + "outputs": [], + "source": [ + "predictions = probability_model.predict(test_images)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "specified-marshall", + "metadata": {}, + "outputs": [], + "source": [ + "def plot_image(i, predictions_array, true_label, img):\n", + " true_label, img = true_label[i], img[i]\n", + " plt.grid(False)\n", + " plt.xticks([])\n", + " plt.yticks([])\n", + "\n", + " plt.imshow(img, cmap=plt.cm.binary)\n", + "\n", + " predicted_label = np.argmax(predictions_array)\n", + " if predicted_label == true_label:\n", + " color = 'blue'\n", + " else:\n", + " color = 'red'\n", + "\n", + " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", + " 100*np.max(predictions_array),\n", + " class_names[true_label]),\n", + " color=color)\n", + "\n", + "def plot_value_array(i, predictions_array, true_label):\n", + " true_label = true_label[i]\n", + " plt.grid(False)\n", + " plt.xticks(range(10))\n", + " plt.yticks([])\n", + " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", + " plt.ylim([0, 1])\n", + " predicted_label = np.argmax(predictions_array)\n", + "\n", + " thisplot[predicted_label].set_color('red')\n", + " thisplot[true_label].set_color('blue')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "prompt-toddler", + "metadata": {}, + "outputs": [], + "source": [ + "i = 0\n", + "plt.figure(figsize=(6,3))\n", + "plt.subplot(1,2,1)\n", + "plot_image(i, predictions[i], test_labels, test_images)\n", + "plt.subplot(1,2,2)\n", + "plot_value_array(i, predictions[i], test_labels)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "naval-concentrate", + "metadata": {}, + "outputs": [], + "source": [ + "i = 12\n", + "plt.figure(figsize=(6,3))\n", + "plt.subplot(1,2,1)\n", + "plot_image(i, predictions[i], test_labels, test_images)\n", + "plt.subplot(1,2,2)\n", + "plot_value_array(i, predictions[i], test_labels)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "amended-ethiopia", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot the first X test images, their predicted labels, and the true labels.\n", + "# Color correct predictions in blue and incorrect predictions in red.\n", + "num_rows = 5\n", + "num_cols = 3\n", + "num_images = num_rows*num_cols\n", + "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", + "for i in range(num_images):\n", + " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", + " plot_image(i, predictions[i], test_labels, test_images)\n", + " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", + " plot_value_array(i, predictions[i], test_labels)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "rental-attribute", + "metadata": {}, + "outputs": [], + "source": [ + "# Grab an image from the test dataset.\n", + "img = test_images[1]\n", + "\n", + "print(img.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "original-architect", + "metadata": {}, + "outputs": [], + "source": [ + "# Add the image to a batch where it's the only member.\n", + "img = (np.expand_dims(img,0))\n", + "\n", + "print(img.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "color-qatar", + "metadata": {}, + "outputs": [], + "source": [ + "predictions_single = probability_model.predict(img)\n", + "\n", + "print(predictions_single)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "pretty-channel", + "metadata": {}, + "outputs": [], + "source": [ + "plot_value_array(1, predictions_single[0], test_labels)\n", + "_ = plt.xticks(range(10), class_names, rotation=45)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "small-emission", + "metadata": {}, + "outputs": [], + "source": [ + "np.argmax(predictions_single[0])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}