From 2f1b6e2a36c5bd910e559fad70e4cef2df6c6cea Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 24 Sep 2024 15:18:35 +0200 Subject: [PATCH] Limit max number of items in dataproviders that support pagination to 10000. Clients can always request more data via offset requests, but since this is not a streaming API we need to build the response in memory before we can start sending. --- client/src/api/schema/schema.ts | 8 ++++++-- lib/galaxy/datatypes/dataproviders/base.py | 14 ++++++------- lib/galaxy/webapps/galaxy/api/datasets.py | 24 +++++++++++++++++++--- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 0659ad0d08d..0e1e42008a6 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -127,7 +127,7 @@ export interface paths { "/api/datasets/{dataset_id}": { /** * Displays information about and/or content of a dataset. - * @description **Note**: Due to the multipurpose nature of this endpoint, which can receive a wild variety of parameters + * @description **Note**: Due to the multipurpose nature of this endpoint, which can receive a wide variety of parameters * and return different kinds of responses, the documentation here will be limited. * To get more information please check the source code. */ @@ -14366,18 +14366,22 @@ export interface operations { show_api_datasets__dataset_id__get: { /** * Displays information about and/or content of a dataset. - * @description **Note**: Due to the multipurpose nature of this endpoint, which can receive a wild variety of parameters + * @description **Note**: Due to the multipurpose nature of this endpoint, which can receive a wide variety of parameters * and return different kinds of responses, the documentation here will be limited. * To get more information please check the source code. */ parameters: { /** @description The type of information about the dataset to be requested. */ /** @description The type of information about the dataset to be requested. Each of these values may require additional parameters in the request and may return different responses. */ + /** @description Maximum number of items to return. Currently only applies to `data_type=raw_data` requests */ + /** @description Starts at the beginning skip the first ( offset - 1 ) items and begin returning at the Nth item. Currently only applies to `data_type=raw_data` requests */ /** @description View to be passed to the serializer */ /** @description Comma-separated list of keys to be passed to the serializer */ query?: { hda_ldda?: components["schemas"]["DatasetSourceType"]; data_type?: components["schemas"]["RequestDataType"] | null; + limit?: number | null; + offset?: number | null; view?: string | null; keys?: string | null; }; diff --git a/lib/galaxy/datatypes/dataproviders/base.py b/lib/galaxy/datatypes/dataproviders/base.py index c1239f86b9f..a271437f184 100644 --- a/lib/galaxy/datatypes/dataproviders/base.py +++ b/lib/galaxy/datatypes/dataproviders/base.py @@ -36,6 +36,7 @@ datasets API entry point: Building a giant list by sweeping all possible dprov classes doesn't make sense For now - I'm burying them in the class __init__s - but I don't like that """ +MAX_LIMIT = 10000 # ----------------------------------------------------------------------------- base classes @@ -233,21 +234,20 @@ class LimitedOffsetDataProvider(FilteredDataProvider): settings = {"limit": "int", "offset": "int"} # TODO: may want to squash this into DataProvider - def __init__(self, source, offset=0, limit=None, **kwargs): + def __init__(self, source, offset=0, limit=MAX_LIMIT, **kwargs): """ :param offset: the number of data to skip before providing. :param limit: the final number of data to provide. """ super().__init__(source, **kwargs) - # how many valid data to skip before we start outputing data - must be positive - # (diff to support neg. indeces - must be pos.) - self.offset = max(offset, 0) + # how many valid data to skip before we start outputting data - must be positive + self.offset = offset - # how many valid data to return - must be positive (None indicates no limit) + # how many valid data to return - must be positive + if limit is None: + limit = MAX_LIMIT self.limit = limit - if self.limit is not None: - self.limit = max(self.limit, 0) def __iter__(self): """ diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 53006b4ee8c..9859f7a7b68 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -29,6 +29,7 @@ from starlette.responses import ( ) from typing_extensions import Annotated +from galaxy.datatypes.dataproviders.base import MAX_LIMIT from galaxy.schema import ( FilterQueryParams, SerializationParams, @@ -432,18 +433,35 @@ class FastAPIDatasets: "may return different responses." ), ), + limit: Annotated[ + Optional[int], + Query( + ge=1, + le=MAX_LIMIT, + description="Maximum number of items to return. Currently only applies to `data_type=raw_data` requests", + ), + ] = MAX_LIMIT, + offset: Annotated[ + Optional[int], + Query( + ge=0, + description="Starts at the beginning skip the first ( offset - 1 ) items and begin returning at the Nth item. Currently only applies to `data_type=raw_data` requests", + ), + ] = 0, serialization_params: SerializationParams = Depends(query_serialization_params), ): """ - **Note**: Due to the multipurpose nature of this endpoint, which can receive a wild variety of parameters + **Note**: Due to the multipurpose nature of this endpoint, which can receive a wide variety of parameters and return different kinds of responses, the documentation here will be limited. To get more information please check the source code. """ - exclude_params = {"hda_ldda", "data_type"} + exclude_params = {"hda_ldda", "data_type", "limit", "offset"} exclude_params.update(SerializationParams.model_fields.keys()) extra_params = get_query_parameters_from_request_excluding(request, exclude_params) - return self.service.show(trans, dataset_id, hda_ldda, serialization_params, data_type, **extra_params) + return self.service.show( + trans, dataset_id, hda_ldda, serialization_params, data_type, limit=limit, offset=offset, **extra_params + ) @router.get( "/api/datasets/{dataset_id}/content/{content_type}",