From c118dd5392a7bb0b8df5dd854556fdfb5215d64e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 11 Jul 2022 12:03:19 +0300 Subject: [PATCH 1/6] Improve displayable function --- lib/galaxy/datatypes/interval.py | 6 ++++-- lib/galaxy/datatypes/tabular.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 03884222b11..b6a13907861 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -27,6 +27,7 @@ from galaxy.datatypes.util.gff_util import ( parse_gff3_attributes, parse_gff_attributes, ) +from galaxy.model import DatasetInstance from galaxy.util import compression_utils from . import ( data, @@ -167,10 +168,11 @@ class Interval(Tabular): else: empty_line_count += 1 - def displayable(self, dataset): + def displayable(self, dataset: DatasetInstance): try: return ( - dataset.has_data() + not dataset.dataset.purged + and dataset.has_data() and dataset.state == dataset.states.OK and dataset.metadata.columns > 0 and dataset.metadata.data_lines != 0 diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index dcf253a2dc0..23c82defe3f 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -34,6 +34,7 @@ from galaxy.datatypes.sniff import ( iter_headers, validate_tabular, ) +from galaxy.model import DatasetInstance from galaxy.util import compression_utils from . import dataproviders @@ -90,10 +91,11 @@ class TabularData(data.Text): if dataset.metadata.comment_lines: dataset.blurb = f"{dataset.blurb}, {util.commaify(str(dataset.metadata.comment_lines))} comments" - def displayable(self, dataset): + def displayable(self, dataset: DatasetInstance): try: return ( - dataset.has_data() + not dataset.dataset.purged + and dataset.has_data() and dataset.state == dataset.states.OK and dataset.metadata.columns > 0 and dataset.metadata.data_lines != 0 From aebc4968ec4531d3a41d58e17847d54956962108 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 11 Jul 2022 12:48:44 +0300 Subject: [PATCH 2/6] Don't attempt to display discarded and deferred datasets Fixes https://sentry.galaxyproject.org/share/issue/395cb0fc507e400fb2f55048d5f7b42e/: ``` FileNotFoundError: [Errno 2] No such file or directory: '' File "/cvmfs/main.galaxyproject.org/venv/lib/python3.8/site-packages/paste/recursive.py", line 85, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/middleware/statsd.py", line 29, in __call__ req = self.application(environ, start_response) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.8/site-packages/paste/httpexceptions.py", line 640, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/base.py", line 159, in __call__ return self.handle_request(environ, start_response) File "galaxy/web/framework/base.py", line 244, in handle_request body = method(trans, **kwargs) File "galaxy/webapps/galaxy/controllers/dataset.py", line 213, in display display_data, headers = data.datatype.display_data( File "galaxy/datatypes/tabular.py", line 164, in display_data chunk=self.get_chunk(trans, dataset, 0), File "galaxy/datatypes/tabular.py", line 105, in get_chunk with compression_utils.get_fileobj(dataset.file_name) as f: File "galaxy/util/compression_utils.py", line 66, in get_fileobj return get_fileobj_raw(filename, mode, compressed_formats)[1] File "galaxy/util/compression_utils.py", line 126, in get_fileobj_raw return compressed_format, open(filename, mode, encoding="utf-8") ``` --- lib/galaxy/webapps/galaxy/controllers/dataset.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 8e7c54cba72..a8b6a51e70b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -25,6 +25,7 @@ from galaxy.managers.hdas import ( HDAManager, ) from galaxy.managers.histories import HistoryManager +from galaxy.model import Dataset from galaxy.model.item_attrs import ( UsesAnnotations, UsesItemRatings, @@ -152,12 +153,18 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE return trans.show_error_message("You are not allowed to access this dataset") if data.purged: return trans.show_error_message("The dataset you are attempting to view has been purged.") - if data.deleted and not (trans.user_is_admin or (data.history and trans.get_user() == data.history.user)): + elif data.deleted and not (trans.user_is_admin or (data.history and trans.get_user() == data.history.user)): return trans.show_error_message("The dataset you are attempting to view has been deleted.") - if data.state == trans.model.Dataset.states.UPLOAD: + elif data.state == Dataset.states.UPLOAD: return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." ) + elif data.state == Dataset.states.DISCARDED: + return trans.show_error_message("The dataset you are attempting to view has been discarded.") + elif data.state == Dataset.states.DEFERRED: + return trans.show_error_message( + "The dataset you are attempting to view has deferred data. You can only use this dataset as input for jobs." + ) return data @web.expose From ebc74c7a02899ab8d4089bdb87d30a92789171b9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 11 Jul 2022 13:20:55 +0300 Subject: [PATCH 3/6] Use _check_dataset in display_username_by_slug in dataset controller Fixes https://sentry.galaxyproject.org/share/issue/77cd5728ab4e4911a62db7a08a09dbee/: ``` FileNotFoundError: [Errno 2] No such file or directory: '' File "uvicorn/protocols/http/h11_impl.py", line 366, in run_asgi result = await app(self.scope, self.receive, self.send) File "uvicorn/middleware/proxy_headers.py", line 75, in __call__ return await self.app(scope, receive, send) File "fastapi/applications.py", line 269, in __call__ await super().__call__(scope, receive, send) File "starlette/applications.py", line 124, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/middleware/errors.py", line 184, in __call__ raise exc File "starlette/middleware/errors.py", line 162, in __call__ await self.app(scope, receive, _send) File "starlette/middleware/base.py", line 69, in __call__ await response(scope, receive, send) File "starlette/responses.py", line 260, in __call__ await wrap(partial(self.listen_for_disconnect, receive)) File "anyio/_backends/_asyncio.py", line 662, in __aexit__ raise exceptions[0] File "starlette/responses.py", line 256, in wrap await func() File "starlette/responses.py", line 245, in stream_response async for chunk in self.body_iterator: File "starlette/middleware/base.py", line 58, in body_stream raise app_exc File "starlette/middleware/base.py", line 36, in coro await self.app(scope, request.receive, send_stream.send) File "starlette_context/middleware/raw_middleware.py", line 96, in __call__ await self.app(scope, receive, send_wrapper) File "starlette/middleware/base.py", line 69, in __call__ await response(scope, receive, send) File "starlette/responses.py", line 260, in __call__ await wrap(partial(self.listen_for_disconnect, receive)) File "anyio/_backends/_asyncio.py", line 662, in __aexit__ raise exceptions[0] File "starlette/responses.py", line 256, in wrap await func() File "starlette/responses.py", line 245, in stream_response async for chunk in self.body_iterator: File "starlette/middleware/base.py", line 58, in body_stream raise app_exc File "starlette/middleware/base.py", line 36, in coro await self.app(scope, request.receive, send_stream.send) File "starlette/middleware/base.py", line 69, in __call__ await response(scope, receive, send) File "starlette/responses.py", line 260, in __call__ await wrap(partial(self.listen_for_disconnect, receive)) File "anyio/_backends/_asyncio.py", line 662, in __aexit__ raise exceptions[0] File "starlette/responses.py", line 256, in wrap await func() File "starlette/responses.py", line 245, in stream_response async for chunk in self.body_iterator: File "starlette/middleware/base.py", line 58, in body_stream raise app_exc File "starlette/middleware/base.py", line 36, in coro await self.app(scope, request.receive, send_stream.send) File "starlette/exceptions.py", line 93, in __call__ raise exc File "starlette/exceptions.py", line 82, in __call__ await self.app(scope, receive, sender) File "fastapi/middleware/asyncexitstack.py", line 21, in __call__ raise e File "fastapi/middleware/asyncexitstack.py", line 18, in __call__ await self.app(scope, receive, send) File "starlette/routing.py", line 670, in __call__ await route.handle(scope, receive, send) File "starlette/routing.py", line 418, in handle await self.app(scope, receive, send) File "a2wsgi/wsgi.py", line 140, in __call__ return await responder(scope, receive, send) File "a2wsgi/wsgi.py", line 179, in __call__ raise self.exc_info[0].with_traceback( File "galaxy/web/framework/middleware/error.py", line 165, in __call__ app_iter = self.application(environ, sr_checker) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.8/site-packages/paste/recursive.py", line 85, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/middleware/statsd.py", line 29, in __call__ req = self.application(environ, start_response) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.8/site-packages/paste/httpexceptions.py", line 640, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/base.py", line 159, in __call__ return self.handle_request(environ, start_response) File "galaxy/web/framework/base.py", line 244, in handle_request body = method(trans, **kwargs) File "galaxy/webapps/galaxy/controllers/dataset.py", line 592, in display_by_username_and_slug first_chunk = dataset.datatype.get_chunk(trans, dataset, 0) File "galaxy/datatypes/tabular.py", line 105, in get_chunk with compression_utils.get_fileobj(dataset.file_name) as f: File "galaxy/util/compression_utils.py", line 66, in get_fileobj return get_fileobj_raw(filename, mode, compressed_formats)[1] File "galaxy/util/compression_utils.py", line 126, in get_fileobj_raw return compressed_format, open(filename, mode, encoding="utf-8") ``` --- .../webapps/galaxy/controllers/dataset.py | 88 +++++++++---------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index a8b6a51e70b..a90cb46c430 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -148,7 +148,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE except Exception: data = None if not data: - raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable(f"Invalid reference dataset id: {str(hda_id)}.") + raise web.httpexceptions.HTTPNotFound(f"Invalid reference dataset id: {str(hda_id)}.") if not self._can_access_dataset(trans, data): return trans.show_error_message("You are not allowed to access this dataset") if data.purged: @@ -570,55 +570,51 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE @web.expose def display_by_username_and_slug(self, trans, username, slug, filename=None, preview=True): """Display dataset by username and slug; because datasets do not yet have slugs, the slug is the dataset's id.""" - id = slug - decoded_id = self.decode_id(id) - dataset = self.hda_manager.get_accessible(decoded_id, trans.user) - dataset = self.hda_manager.error_if_uploading(dataset) - if dataset: - # Filename used for composite types. - if filename: - return self.display(trans, dataset_id=slug, filename=filename) + dataset = self._check_dataset(trans, slug) + if not isinstance(dataset, trans.app.model.DatasetInstance): + return dataset + # Filename used for composite types. + if filename: + return self.display(trans, dataset_id=slug, filename=filename) - truncated, dataset_data = self.hda_manager.text_data(dataset, preview) - dataset.annotation = self.get_item_annotation_str(trans.sa_session, dataset.history.user, dataset) + truncated, dataset_data = self.hda_manager.text_data(dataset, preview) + dataset.annotation = self.get_item_annotation_str(trans.sa_session, dataset.history.user, dataset) - # If dataset is chunkable, get first chunk. - first_chunk = None - if dataset.datatype.CHUNKABLE: - first_chunk = dataset.datatype.get_chunk(trans, dataset, 0) + # If dataset is chunkable, get first chunk. + first_chunk = None + if dataset.datatype.CHUNKABLE: + first_chunk = dataset.datatype.get_chunk(trans, dataset, 0) - # If data is binary or an image, stream without template; otherwise, use display template. - # TODO: figure out a way to display images in display template. - if ( - isinstance(dataset.datatype, datatypes.binary.Binary) - or isinstance(dataset.datatype, datatypes.images.Image) - or isinstance(dataset.datatype, datatypes.text.Html) - ): - trans.response.set_content_type(dataset.get_mime()) - return open(dataset.file_name, "rb") - else: - # Get rating data. - user_item_rating = 0 - if trans.get_user(): - user_item_rating = self.get_user_item_rating(trans.sa_session, trans.get_user(), dataset) - if user_item_rating: - user_item_rating = user_item_rating.rating - else: - user_item_rating = 0 - ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, dataset) - - return trans.fill_template_mako( - "/dataset/display.mako", - item=dataset, - item_data=dataset_data, - truncated=truncated, - user_item_rating=user_item_rating, - ave_item_rating=ave_item_rating, - num_ratings=num_ratings, - first_chunk=first_chunk, - ) + # If data is binary or an image, stream without template; otherwise, use display template. + # TODO: figure out a way to display images in display template. + if ( + isinstance(dataset.datatype, datatypes.binary.Binary) + or isinstance(dataset.datatype, datatypes.images.Image) + or isinstance(dataset.datatype, datatypes.text.Html) + ): + trans.response.set_content_type(dataset.get_mime()) + return open(dataset.file_name, "rb") else: - raise web.httpexceptions.HTTPNotFound() + # Get rating data. + user_item_rating = 0 + if trans.get_user(): + user_item_rating = self.get_user_item_rating(trans.sa_session, trans.get_user(), dataset) + if user_item_rating: + user_item_rating = user_item_rating.rating + else: + user_item_rating = 0 + ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, dataset) + + return trans.fill_template_mako( + "/dataset/display.mako", + item=dataset, + item_data=dataset_data, + truncated=truncated, + user_item_rating=user_item_rating, + ave_item_rating=ave_item_rating, + num_ratings=num_ratings, + first_chunk=first_chunk, + ) @web.expose def get_item_content_async(self, trans, id): From 66180e2a85a1766777d3086f0726dfbadae99b9f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 11 Jul 2022 13:30:14 +0300 Subject: [PATCH 4/6] Do URL encoding of workflow slug --- client/src/components/Workflow/WorkflowDropdown.vue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/components/Workflow/WorkflowDropdown.vue b/client/src/components/Workflow/WorkflowDropdown.vue index 013723146f6..ff1143d5b93 100644 --- a/client/src/components/Workflow/WorkflowDropdown.vue +++ b/client/src/components/Workflow/WorkflowDropdown.vue @@ -101,9 +101,9 @@ export default { return `${getAppRoot()}workflows/${this.workflow.id}/invocations`; }, urlViewShared() { - return `${getAppRoot()}workflow/display_by_username_and_slug?username=${this.workflow.owner}&slug=${ - this.workflow.slug - }`; + return `${getAppRoot()}workflow/display_by_username_and_slug?username=${ + this.workflow.owner + }&slug=${encodeURIComponent(this.workflow.slug)}`; }, readOnly() { return !!this.workflow.shared; From ec39bd302cb7149db724ade49ff54106a0845197 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 11 Jul 2022 18:09:29 +0300 Subject: [PATCH 5/6] Debug collection validation error --- .../webapps/galaxy/services/dataset_collections.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/services/dataset_collections.py b/lib/galaxy/webapps/galaxy/services/dataset_collections.py index 8169674cbe6..a3ffb62cf31 100644 --- a/lib/galaxy/webapps/galaxy/services/dataset_collections.py +++ b/lib/galaxy/webapps/galaxy/services/dataset_collections.py @@ -9,6 +9,7 @@ from pydantic import ( BaseModel, Extra, Field, + ValidationError, ) from galaxy import exceptions @@ -254,4 +255,10 @@ class DatasetCollectionsService(ServiceBase, UsesLibraryMixinItems): return result rval = [serialize_element(el) for el in contents] - return DatasetCollectionContentElements.parse_obj(rval) + try: + return DatasetCollectionContentElements.parse_obj(rval) + except ValidationError: + log.exception( + f"Serializing DatasetCollectionContentsElements failed. Collection is populated: {hdca.collection.populated}" + ) + raise From 75470b132299f79d1db4d02517777d7b587f0a6d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 12 Jul 2022 13:40:55 +0300 Subject: [PATCH 6/6] Prevent viewing datasets in paused state Fixes https://sentry.galaxyproject.org/share/issue/4fb1a1d19d9d41f89b8efc6b3876b599/: ``` FileNotFoundError: [Errno 2] No such file or directory: '' File "uvicorn/protocols/http/h11_impl.py", line 366, in run_asgi result = await app(self.scope, self.receive, self.send) File "uvicorn/middleware/proxy_headers.py", line 75, in __call__ return await self.app(scope, receive, send) File "fastapi/applications.py", line 269, in __call__ await super().__call__(scope, receive, send) File "starlette/applications.py", line 124, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/middleware/errors.py", line 184, in __call__ raise exc File "starlette/middleware/errors.py", line 162, in __call__ await self.app(scope, receive, _send) File "starlette/middleware/base.py", line 69, in __call__ await response(scope, receive, send) File "starlette/responses.py", line 260, in __call__ await wrap(partial(self.listen_for_disconnect, receive)) File "anyio/_backends/_asyncio.py", line 662, in __aexit__ raise exceptions[0] File "starlette/responses.py", line 256, in wrap await func() File "starlette/responses.py", line 245, in stream_response async for chunk in self.body_iterator: File "starlette/middleware/base.py", line 58, in body_stream raise app_exc File "starlette/middleware/base.py", line 36, in coro await self.app(scope, request.receive, send_stream.send) File "starlette_context/middleware/raw_middleware.py", line 96, in __call__ await self.app(scope, receive, send_wrapper) File "starlette/middleware/base.py", line 69, in __call__ await response(scope, receive, send) File "starlette/responses.py", line 260, in __call__ await wrap(partial(self.listen_for_disconnect, receive)) File "anyio/_backends/_asyncio.py", line 662, in __aexit__ raise exceptions[0] File "starlette/responses.py", line 256, in wrap await func() File "starlette/responses.py", line 245, in stream_response async for chunk in self.body_iterator: File "starlette/middleware/base.py", line 58, in body_stream raise app_exc File "starlette/middleware/base.py", line 36, in coro await self.app(scope, request.receive, send_stream.send) File "starlette/middleware/base.py", line 69, in __call__ await response(scope, receive, send) File "starlette/responses.py", line 260, in __call__ await wrap(partial(self.listen_for_disconnect, receive)) File "anyio/_backends/_asyncio.py", line 662, in __aexit__ raise exceptions[0] File "starlette/responses.py", line 256, in wrap await func() File "starlette/responses.py", line 245, in stream_response async for chunk in self.body_iterator: File "starlette/middleware/base.py", line 58, in body_stream raise app_exc File "starlette/middleware/base.py", line 36, in coro await self.app(scope, request.receive, send_stream.send) File "starlette/exceptions.py", line 93, in __call__ raise exc File "starlette/exceptions.py", line 82, in __call__ await self.app(scope, receive, sender) File "fastapi/middleware/asyncexitstack.py", line 21, in __call__ raise e File "fastapi/middleware/asyncexitstack.py", line 18, in __call__ await self.app(scope, receive, send) File "starlette/routing.py", line 670, in __call__ await route.handle(scope, receive, send) File "starlette/routing.py", line 418, in handle await self.app(scope, receive, send) File "a2wsgi/wsgi.py", line 140, in __call__ return await responder(scope, receive, send) File "a2wsgi/wsgi.py", line 179, in __call__ raise self.exc_info[0].with_traceback( File "galaxy/web/framework/middleware/error.py", line 165, in __call__ app_iter = self.application(environ, sr_checker) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.8/site-packages/paste/recursive.py", line 85, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/middleware/statsd.py", line 29, in __call__ req = self.application(environ, start_response) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.8/site-packages/paste/httpexceptions.py", line 640, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/base.py", line 159, in __call__ return self.handle_request(environ, start_response) File "galaxy/web/framework/base.py", line 244, in handle_request body = method(trans, **kwargs) File "galaxy/webapps/galaxy/controllers/dataset.py", line 209, in display display_data, headers = data.datatype.display_data( File "galaxy/datatypes/sequence.py", line 753, in display_data with compression_utils.get_fileobj(dataset.file_name) as fh: File "galaxy/util/compression_utils.py", line 66, in get_fileobj return get_fileobj_raw(filename, mode, compressed_formats)[1] File "galaxy/util/compression_utils.py", line 126, in get_fileobj_raw return compressed_format, open(filename, mode, encoding="utf-8") ``` --- lib/galaxy/webapps/galaxy/controllers/dataset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index a90cb46c430..ec8d8df55b5 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -165,6 +165,10 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE return trans.show_error_message( "The dataset you are attempting to view has deferred data. You can only use this dataset as input for jobs." ) + elif data.state == Dataset.states.PAUSED: + return trans.show_error_message( + "The dataset you are attempting to view is in paused state. One of the inputs for the job that creates this dataset has failed." + ) return data @web.expose