diff --git a/client/src/bundleEntries.js b/client/src/bundleEntries.js index 6f344609971..19f6a629f5a 100644 --- a/client/src/bundleEntries.js +++ b/client/src/bundleEntries.js @@ -99,6 +99,7 @@ export { mountPageEditor } from "components/PageEditor/mount"; export { mountPageDisplay } from "components/PageDisplay"; export { mountDestinationParams } from "components/JobDestinationParams"; export { mountDatasetInformation } from "components/DatasetInformation"; +export { mountDatasetStorage } from "components/Dataset/DatasetStorage"; // Used in common.mako export { default as store } from "storemodern"; diff --git a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue new file mode 100644 index 00000000000..cbbd6652a1b --- /dev/null +++ b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue @@ -0,0 +1,69 @@ + + + diff --git a/client/src/components/Dataset/DatasetStorage/index.js b/client/src/components/Dataset/DatasetStorage/index.js new file mode 100644 index 00000000000..b2cedf03e1b --- /dev/null +++ b/client/src/components/Dataset/DatasetStorage/index.js @@ -0,0 +1,4 @@ +export { default as DatasetStorage } from "./DatasetStorage"; + +// functions for mounting job metrics in non-Vue environments +export { mountDatasetStorage } from "./mount"; diff --git a/client/src/components/Dataset/DatasetStorage/mount.js b/client/src/components/Dataset/DatasetStorage/mount.js new file mode 100644 index 00000000000..78224bb6783 --- /dev/null +++ b/client/src/components/Dataset/DatasetStorage/mount.js @@ -0,0 +1,16 @@ +/** + * Endpoint for mounting job metrics from non-Vue environment. + */ +import DatasetStorage from "./DatasetStorage.vue"; +import { mountVueComponent } from "utils/mountVueComponent"; + +export const mountDatasetStorage = (propsData = {}) => { + const elements = document.querySelectorAll(".dataset-storage"); + Array.prototype.forEach.call(elements, function (el, i) { + const datasetId = el.getAttribute("dataset_id"); + const datasetType = el.getAttribute("dataset_type") || "hda"; + propsData.datasetId = datasetId; + propsData.datasetType = datasetType; + mountVueComponent(DatasetStorage)(propsData, el); + }); +}; diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 2166b2a8120..d752af00c5e 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -171,12 +171,42 @@ class ObjectStore(metaclass=abc.ABCMeta): @abc.abstractmethod def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ - Return the URL for direct acces if supported, otherwise return None. + Return the URL for direct access if supported, otherwise return None. Note: need to be careful to not bypass dataset security with this. """ raise NotImplementedError() + @abc.abstractmethod + def get_concrete_store_name(self, obj): + """Return a display name or title of the objectstore corresponding to obj. + + To accommodate nested objectstores, obj is passed in so this metadata can + be returned for the ConcreteObjectStore corresponding to the object. + """ + + @abc.abstractmethod + def get_concrete_store_description_markdown(self, obj): + """Return a longer description of how data 'obj' is stored. + + To accommodate nested objectstores, obj is passed in so this metadata can + be returned for the ConcreteObjectStore corresponding to the object. + """ + + @abc.abstractmethod + def is_transient(self, obj): + """Return boolean indicating if obj should be treated as transient. + + To accommodate nested objectstores, obj is passed in so this metadata can + be returned for the ConcreteObjectStore corresponding to the object. + + ObjectStores corresponding to data sources that get purged frequently + can mark that here. In the future it would be nice to provide different + icons in the UI for instance based on this or warn people before + publishing pages with links to transient datasets for instance, this + should provide a piece of the puzzle for doing that in the future. + """ + @abc.abstractmethod def get_store_usage_percent(self): """Return the percentage indicating how full the store is.""" @@ -292,6 +322,15 @@ class BaseObjectStore(ObjectStore): def get_object_url(self, obj, **kwargs): return self._invoke('get_object_url', obj, **kwargs) + def get_concrete_store_name(self, obj): + return self._invoke('get_concrete_store_name', obj) + + def get_concrete_store_description_markdown(self, obj): + return self._invoke('get_concrete_store_description_markdown', obj) + + def is_transient(self, obj): + return self._invoke('is_transient', obj) + def get_store_usage_percent(self): return self._invoke('get_store_usage_percent') @@ -323,12 +362,27 @@ class ConcreteObjectStore(BaseObjectStore): config_dict = {} super().__init__(config=config, config_dict=config_dict, **kwargs) self.store_by = config_dict.get("store_by", None) or getattr(config, "object_store_store_by", "id") + self.name = config_dict.get("name", None) + self.description = config_dict.get("description", None) + self.transient = config_dict.get("transient", False) def to_dict(self): rval = super().to_dict() rval["store_by"] = self.store_by + rval["name"] = self.name + rval["description"] = self.description + rval["transient"] = self.transient return rval + def _get_concrete_store_name(self, obj): + return self.name + + def _get_concrete_store_description_markdown(self, obj): + return self.description + + def _is_transient(self, obj): + return self.transient + def _get_store_by(self, obj): return self.store_by @@ -378,9 +432,17 @@ class DiskObjectStore(ConcreteObjectStore): store_by = config_xml.attrib.get('store_by', None) if store_by is not None: config_dict['store_by'] = store_by + transient = config_xml.attrib.get('transient', None) + if transient is not None: + config_dict['transient'] = asbool(transient) + name = config_xml.attrib.get('name', None) + if name is not None: + config_dict['name'] = name for e in config_xml: if e.tag == 'files_dir': config_dict["files_dir"] = e.get('path') + elif e.tag == 'description': + config_dict["description"] = e.text else: extra_dirs.append({"type": e.get('type'), "path": e.get('path')}) @@ -659,6 +721,15 @@ class NestedObjectStore(BaseObjectStore): """For the first backend that has this `obj`, get its URL.""" return self._call_method('_get_object_url', obj, None, False, **kwargs) + def _get_concrete_store_name(self, obj): + return self._call_method('_get_concrete_store_name', obj, None, True) + + def _get_concrete_store_description_markdown(self, obj): + return self._call_method('_get_concrete_store_description_markdown', obj, None, True) + + def _is_transient(self, obj): + return self._call_method('_is_transient', obj, None, True) + def _get_store_by(self, obj): return self._call_method('_get_store_by', obj, None, False) diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index fcefc974743..9069bb1139d 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -155,6 +155,36 @@ class DatasetsController(BaseAPIController, UsesVisualizationMixin): rval = dataset.to_dict() return rval + @web.expose_api_anonymous + def show_storage(self, trans, dataset_id, hda_ldda='hda', **kwd): + """ + GET /api/datasets/{encoded_dataset_id}/storage + + Display user-facing storage details related to the objectstore a + dataset resides in. + """ + dataset_instance = self.get_hda_or_ldda(trans, hda_ldda=hda_ldda, dataset_id=dataset_id) + dataset = dataset_instance.dataset + object_store = self.app.object_store + object_store_id = dataset.object_store_id + name = object_store.get_concrete_store_name(dataset) + description = object_store.get_concrete_store_description_markdown(dataset) + transient = object_store.is_transient(dataset) + # not really working (existing problem) + try: + percent_used = object_store.get_store_usage_percent() + except AttributeError: + # not implemented on nestedobjectstores yet. + percent_used = None + + return { + 'object_store_id': object_store_id, + 'name': name, + 'description': description, + 'transient': transient, + 'percent_used': percent_used, + } + @web.expose_api def update_permissions(self, trans, dataset_id, payload, **kwd): """ diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 23792168bd9..cde0cc4ef4a 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -264,6 +264,11 @@ def populate_api_routes(webapp, app): controller="datasets", action="get_content_as_text", conditions=dict(method=["GET"])) + webapp.mapper.connect('dataset_storage', + '/api/datasets/{dataset_id}/storage', + controller='datasets', + action='show_storage', + conditions=dict(method=["GET"])) webapp.mapper.connect("history_contents_metadata_file", "/api/histories/{history_id}/contents/{history_content_id}/metadata_file", controller="datasets", diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 075222d1d63..0aa91b6f372 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -513,6 +513,11 @@ class BaseDatasetPopulator: src = 'hdca' return dict(src=src, id=history_content["id"]) + def dataset_storage_info(self, dataset_id): + storage_response = self.galaxy_interactor.get("datasets/{}/storage".format(dataset_id)) + storage_response.raise_for_status() + return storage_response.json() + def get_roles(self): roles_response = self.galaxy_interactor.get("roles", admin=True) assert roles_response.status_code == 200 diff --git a/templates/show_params.mako b/templates/show_params.mako index 703f9184187..d58b62d4082 100644 --- a/templates/show_params.mako +++ b/templates/show_params.mako @@ -85,6 +85,9 @@ encoded_history_id = trans.security.encode_id( hda.history_id ) +
+
+ %if job:
@@ -175,5 +178,6 @@ $(function(){ window.bundleEntries.mountJobParameters(); window.bundleEntries.mountDestinationParams(); window.bundleEntries.mountDatasetInformation(); + window.bundleEntries.mountDatasetStorage(); }); diff --git a/test/integration/objectstore/test_selection.py b/test/integration/objectstore/test_selection.py index dd4c0e3dd28..c3a3a80622e 100644 --- a/test/integration/objectstore/test_selection.py +++ b/test/integration/objectstore/test_selection.py @@ -12,7 +12,8 @@ JOB_RESOURCE_PARAMETERS_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "selection_ DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template(""" - + + This is my description of the default store with *markdown*. @@ -86,6 +87,9 @@ class ObjectStoreSelectionIntegrationTestCase(BaseObjectStoreIntegrationTestCase hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") self.dataset_populator.wait_for_history(history_id) hda1_input = {"src": "hda", "id": hda1["id"]} + storage_info = self.dataset_populator.dataset_storage_info(hda1["id"]) + assert "Default Store" == storage_info["name"] + assert "*markdown*" in storage_info["description"] # One file uploaded, added to default object store ID. self._assert_file_counts(1, 0, 0, 0) diff --git a/test/unit/objectstore/test_objectstore.py b/test/unit/objectstore/test_objectstore.py index 64932e5457f..0f12f70521b 100644 --- a/test/unit/objectstore/test_objectstore.py +++ b/test/unit/objectstore/test_objectstore.py @@ -197,12 +197,20 @@ MIXED_STORE_BY_HIERARCHICAL_TEST_CONFIG = """ HIERARCHICAL_TEST_CONFIG = """ - + + + This is our new storage cluster, check out the storage + on our institute's system page for [Fancy New Storage](http://computecenter.example.com/systems/fancystorage). + - + + + This is our older legacy storage cluster, check out the storage + on our institute's system page for [Legacy Storage](http://computecenter.example.com/systems/legacystorage). + @@ -215,6 +223,10 @@ HIERARCHICAL_TEST_CONFIG_YAML = """ type: hierarchical backends: - id: files1 + name: Newer Cool Storage + description: | + This is our new storage cluster, check out the storage + on our institute's system page for [Fancy New Storage](http://computecenter.example.com/systems/fancystorage). type: disk weight: 1 files_dir: "${temp_directory}/files1" @@ -224,6 +236,10 @@ backends: - type: job_work path: "${temp_directory}/job_working_directory1" - id: files2 + name: Older Legacy Storage + description: | + This is our older legacy storage cluster, check out the storage + on our institute's system page for [Legacy Storage](http://computecenter.example.com/systems/legacystorage). type: disk weight: 1 files_dir: "${temp_directory}/files2" @@ -248,11 +264,21 @@ def test_hierarchical_store(): assert object_store.exists(MockDataset(2)) assert object_store.empty(MockDataset(2)) - # Write non-empty dataset in backend 1, test it is not emtpy & exists. + # Write non-empty dataset in backend 1, test it is not empty & exists. directory.write("Hello World!", "files1/000/dataset_3.dat") assert object_store.exists(MockDataset(3)) assert not object_store.empty(MockDataset(3)) + # check and description routed correctly + files1_desc = object_store.get_concrete_store_description_markdown(MockDataset(3)) + files1_name = object_store.get_concrete_store_name(MockDataset(3)) + files2_desc = object_store.get_concrete_store_description_markdown(MockDataset(2)) + files2_name = object_store.get_concrete_store_name(MockDataset(2)) + assert "fancy" in files1_desc + assert "Newer Cool" in files1_name + assert "older" in files2_desc + assert "Legacy" in files2_name + # Assert creation always happens in first backend. for i in range(100): dataset = MockDataset(100 + i)