From e239118227ca04171cef17d244772379ef2d0847 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Mon, 19 Dec 2022 19:53:54 -0500 Subject: [PATCH 01/15] Filter Inputs and Outputs in HistoryPanel --- .../History/Content/ContentItem.vue | 28 ++-- .../HistoryFilters/HistoryFilters.vue | 2 + .../History/CurrentHistory/HistoryPanel.vue | 32 ++--- .../src/components/History/HistoryFilters.js | 1 + .../store/historyStore/historyItemsStore.js | 100 ++++++++++++++ .../src/stores/history/historyItemsStore.js | 11 +- lib/galaxy/managers/history_contents.py | 4 + lib/galaxy/managers/jobs.py | 126 ++++++++++++++++++ .../galaxy/services/history_contents.py | 17 +++ 9 files changed, 289 insertions(+), 32 deletions(-) create mode 100644 client/src/store/historyStore/historyItemsStore.js diff --git a/client/src/components/History/Content/ContentItem.vue b/client/src/components/History/Content/ContentItem.vue index b57282a2e36..0f93ee627d6 100644 --- a/client/src/components/History/Content/ContentItem.vue +++ b/client/src/components/History/Content/ContentItem.vue @@ -18,23 +18,22 @@ - - - + + + @@ -86,7 +85,7 @@ v-if="expandDataset" :dataset="item" :writable="writable" - :show-highlight="isHistoryItem" + :show-highlight="isHistoryItem && filterable" :item-urls="itemUrls" @edit="onEdit" @toggleHighlights="toggleHighlights" /> @@ -104,11 +103,10 @@ import { updateContentFields } from "components/History/model/queries"; import { JobStateSummary } from "./Collection/JobStateSummary"; import { library } from "@fortawesome/fontawesome-svg-core"; import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome"; -import { faArrowCircleUp, faMinusCircle, faCheckCircle } from "@fortawesome/free-solid-svg-icons"; +import { faArrowCircleUp, faArrowCircleDown, faCheckCircle } from "@fortawesome/free-solid-svg-icons"; import { useEntryPointStore } from "stores/entryPointStore"; -library.add(faArrowCircleUp, faMinusCircle, faCheckCircle); - +library.add(faArrowCircleUp, faArrowCircleDown, faCheckCircle); export default { components: { CollectionDescription, diff --git a/client/src/components/History/CurrentHistory/HistoryFilters/HistoryFilters.vue b/client/src/components/History/CurrentHistory/HistoryFilters/HistoryFilters.vue index 5d4a015db72..12a81ffe24b 100644 --- a/client/src/components/History/CurrentHistory/HistoryFilters/HistoryFilters.vue +++ b/client/src/components/History/CurrentHistory/HistoryFilters/HistoryFilters.vue @@ -46,6 +46,8 @@ Filter by state: + Filter by related to item index: + Filter by item index: diff --git a/client/src/components/History/CurrentHistory/HistoryPanel.vue b/client/src/components/History/CurrentHistory/HistoryPanel.vue index 187c6f945d1..4e98ec32fad 100644 --- a/client/src/components/History/CurrentHistory/HistoryPanel.vue +++ b/client/src/components/History/CurrentHistory/HistoryPanel.vue @@ -144,7 +144,6 @@ import { useHistoryItemsStore } from "stores/history/historyItemsStore"; import LoadingSpan from "components/LoadingSpan"; import ContentItem from "components/History/Content/ContentItem"; import { deleteContent, updateContentFields } from "components/History/model/queries"; -import { getHighlights } from "components/History/Content/model/highlights"; import ExpandedItems from "components/History/Content/ExpandedItems"; import SelectedItems from "components/History/Content/SelectedItems"; import ListingLayout from "components/History/Layout/ListingLayout"; @@ -193,7 +192,6 @@ export default { return { error: null, filterText: "", - highlights: {}, highlightsKey: null, invisible: {}, loading: false, @@ -255,13 +253,11 @@ export default { queryKey() { this.invisible = {}; this.offset = 0; - this.resetHighlights(); this.loadHistoryItems(); }, historyId(newVal, oldVal) { if (newVal !== oldVal) { this.operationRunning = null; - this.resetHighlights(); } }, filter(newVal) { @@ -281,7 +277,19 @@ export default { ...vuexMapActions("history", ["loadHistoryById"]), ...mapActions(useHistoryItemsStore, ["fetchHistoryItems"]), getHighlight(item) { - return this.highlights[this.getItemKey(item)]; + if (this.filterText.includes("related:" + item.hid)) { + this.highlightsKey = item.hid; + return "currItem"; + } else if (this.filterText.includes("related:") && this.highlightsKey) { + if (item.hid > this.highlightsKey) { + return "output"; + } else { + return "input"; + } + } else { + this.highlightsKey = null; + return null; + } }, getItemKey(item) { return `${item.id}-${item.history_content_type}`; @@ -344,19 +352,13 @@ export default { console.debug("HistoryPanel - Operation error.", error); this.operationError = error; }, - async toggleHighlights(item) { - const key = this.getItemKey(item); - if (this.highlightsKey != key) { - this.highlightsKey = key; - this.highlights = await getHighlights(item, key); + toggleHighlights(item) { + if (this.filterText == "related:" + item.hid) { + this.filterText = ""; } else { - this.resetHighlights(); + this.filterText = "related:" + item.hid; } }, - resetHighlights() { - this.highlights = {}; - this.highlightsKey = null; - }, onDragEnter(e) { this.dragTarget = e.target; this.showDropZone = true; diff --git a/client/src/components/History/HistoryFilters.js b/client/src/components/History/HistoryFilters.js index f61d53f5883..e8d9dd9bb8e 100644 --- a/client/src/components/History/HistoryFilters.js +++ b/client/src/components/History/HistoryFilters.js @@ -9,6 +9,7 @@ export const validFilters = { hid_gt: compare("hid", "gt"), hid_le: compare("hid", "le"), hid_lt: compare("hid", "lt"), + related: equals("related"), tag: contains("tags", "tag", expandNameTag), visible: equals("visible", "visible", toBool), deleted: equals("deleted", "deleted", toBool), diff --git a/client/src/store/historyStore/historyItemsStore.js b/client/src/store/historyStore/historyItemsStore.js new file mode 100644 index 00000000000..cee4b557d79 --- /dev/null +++ b/client/src/store/historyStore/historyItemsStore.js @@ -0,0 +1,100 @@ +/** + * Requests history items by reacting to changes of filter props passed + * to the history items provider used in the history panel. + */ + +import Vue from "vue"; +import { reverse } from "lodash"; +import { LastQueue } from "utils/promise-queue"; +import { urlData } from "utils/url"; +import { mergeArray } from "./model/utilities"; +import { getFilters, getQueryString, testFilters } from "utils/filterConversion"; + +const limit = 100; +const queue = new LastQueue(); + +const state = { + items: {}, + itemKey: "hid", + latestCreateTime: new Date(), + totalMatchesCount: undefined, + lastCheckedTime: new Date(), + matchedHids: {}, + isWatching: false, +}; + +const getters = { + getHistoryItems: + (state) => + ({ historyId, filterText }) => { + const itemArray = state.items[historyId] || []; + const filters = getFilters(filterText).filter((filter) => !filter.includes("related")); + const filtered = itemArray.filter((item) => { + if (!item) { + return false; + } + if (!testFilters(filters, item)) { + return false; + } + if (!state.matchedHids[historyId].includes(item.hid)) { + return false; + } + return true; + }); + return reverse(filtered); + }, + getLatestCreateTime: (state) => () => state.latestCreateTime, + getTotalMatchesCount: (state) => () => state.totalMatchesCount, + getLastCheckedTime: (state) => () => state.lastCheckedTime, + getWatchingVisibility: (state) => () => state.isWatching, +}; + +const actions = { + fetchHistoryItems: async ({ commit }, { historyId, filterText, offset }) => { + const queryString = getQueryString(filterText); + const params = `v=dev&order=hid&offset=${offset}&limit=${limit}`; + const url = `/api/histories/${historyId}/contents?${params}&${queryString}`; + const headers = { accept: "application/vnd.galaxy.history.contents.stats+json" }; + await queue.enqueue(urlData, { url, headers }, historyId).then((data) => { + const stats = data.stats; + commit("saveQueryStats", { stats }); + const payload = data.contents; + commit("saveHistoryItems", { historyId, payload }); + }); + }, +}; + +const mutations = { + saveHistoryItems: (state, { historyId, payload }) => { + // merges incoming payload into existing state + mergeArray(historyId, payload, state.items, state.itemKey); + const payloadHids = []; + // keep track of latest create time for items + payload.forEach((item) => { + if (item.state == "ok") { + const itemCreateTime = new Date(item.create_time); + if (itemCreateTime > state.latestCreateTime) { + state.latestCreateTime = itemCreateTime; + } + } + payloadHids.push(item.hid); + }); + Vue.set(state.matchedHids, historyId, payloadHids); + }, + setLastCheckedTime: (state, { checkForUpdate }) => { + state.lastCheckedTime = checkForUpdate; + }, + setWatchingVisibility: (state, { watchingVisibility }) => { + state.isWatching = watchingVisibility; + }, + saveQueryStats: (state, { stats }) => { + state.totalMatchesCount = stats.total_matches; + }, +}; + +export const historyItemsStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/src/stores/history/historyItemsStore.js b/client/src/stores/history/historyItemsStore.js index ea2f282193d..fcdbe0a2409 100644 --- a/client/src/stores/history/historyItemsStore.js +++ b/client/src/stores/history/historyItemsStore.js @@ -4,7 +4,7 @@ */ import { defineStore } from "pinia"; - +import Vue from "vue"; import { reverse } from "lodash"; import { LastQueue } from "utils/promise-queue"; import { urlData } from "utils/url"; @@ -21,13 +21,14 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { latestCreateTime: new Date(), totalMatchesCount: undefined, lastCheckedTime: new Date(), + matchedHids: {}, isWatching: false, }), getters: { getHistoryItems: (state) => { return (historyId, filterText) => { const itemArray = state.items[historyId] || []; - const filters = HistoryFilters.getFilters(filterText); + const filters = HistoryFilters.getFilters(filterText).filter((filter) => !filter.includes("related")); const filtered = itemArray.filter((item) => { if (!item) { return false; @@ -35,6 +36,9 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { if (!HistoryFilters.testFilters(filters, item)) { return false; } + if (!state.matchedHids[historyId].includes(item.hid)) { + return false; + } return true; }); return reverse(filtered); @@ -71,6 +75,7 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { this.$patch((state) => { // merges incoming payload into existing state mergeArray(historyId, payload, state.items, state.itemKey); + const payloadHids = []; // keep track of latest create time for items payload.forEach((item) => { if (item.state == "ok") { @@ -79,7 +84,9 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { state.latestCreateTime = itemCreateTime; } } + payloadHids.push(item.hid); }); + Vue.set(state.matchedHids, historyId, payloadHids); }); }, setLastCheckedTime(checkForUpdate) { diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 34131c07af7..6d2d6314d63 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -538,6 +538,9 @@ class HistoryContentsFilters( return sql.column("history_content_type") == val raise_filter_err(attr, op, val, "bad op in filter") + if attr == "related" and op == "eq": + return sql.column("hid").in_(val) + if attr == "type_id": if op == "eq": return sql.column("type_id") == val @@ -595,6 +598,7 @@ class HistoryContentsFilters( self.orm_filter_parsers.update( { "history_content_type": {"op": ("eq")}, + "related": {"op": ("eq")}, "type_id": {"op": ("eq", "in"), "val": self.parse_type_id_list}, "hid": {"op": ("eq", "ge", "le", "gt", "lt"), "val": int}, # TODO: needs a different val parser - but no way to add to the above diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index a32a994b602..c9a9972dc95 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -15,6 +15,7 @@ from sqlalchemy import ( and_, false, func, + literal, or_, ) from sqlalchemy.orm import aliased @@ -31,6 +32,7 @@ from galaxy.job_metrics import ( RawMetric, Safety, ) +from galaxy.managers.base import get_object from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.datasets import DatasetManager from galaxy.managers.hdas import HDAManager @@ -1014,3 +1016,127 @@ def summarize_job_outputs(job: model.Job, tool, params, security): } ) return outputs + + +class JobConnectionsManager: + def __init__(self, sa_session: galaxy_scoped_session): + self.sa_session = sa_session + + def get_related_hids(self, trans, item_data, item_hid): + item_id = item_data[0] + item_type = item_data[1] + results = [item_hid] + + # Get select queries + if item_type == 'HDA': + output_selects = self.outputs_derived_from_input_hda(item_id) + input_selects = self.inputs_for_hda(item_id) + elif item_type == 'HDCA': + output_selects = self.outputs_derived_from_input_hdca(item_id) + input_selects = self.inputs_for_hdca(item_id) + else: + raise Exception(f"Invalid item type {item_type}") + + # Add found related items' hids to results list + for select_query in (output_selects + input_selects): + query_res = self.sa_session.execute(select_query).all() + for res in query_res: + res_item = get_object(trans, res.id, res.src) + results.append(res_item.hid) + return results + + def outputs_derived_from_input_hda(self, input_hda_id): + hda_select = ( + select([literal("HistoryDatasetAssociation").label("src"), model.JobToOutputDatasetAssociation.dataset_id.label("id")]) + .join( + model.JobToInputDatasetAssociation, + model.JobToInputDatasetAssociation.job_id == model.JobToOutputDatasetAssociation.job_id, + ) + .where(model.JobToInputDatasetAssociation.dataset_id == input_hda_id) + ) + hdca_select = ( + select( + [ + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), + ] + ) + .join( + model.JobToInputDatasetAssociation, + model.JobToInputDatasetAssociation.job_id == model.JobToOutputDatasetCollectionAssociation.job_id, + ) + .where(model.JobToInputDatasetAssociation.dataset_id == input_hda_id) + ) + return hda_select, hdca_select + + def outputs_derived_from_input_hdca(self, input_hdca_id): + hda_select = ( + select([literal("HistoryDatasetAssociation").label("src"), model.JobToOutputDatasetAssociation.dataset_id.label("id")]) + .join( + model.JobToInputDatasetCollectionAssociation, + model.JobToInputDatasetCollectionAssociation.job_id == model.JobToOutputDatasetAssociation.job_id, + ) + .where(model.JobToInputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) + ) + hdca_select = ( + select( + [ + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), + ] + ) + .join( + model.JobToInputDatasetCollectionAssociation, + model.JobToInputDatasetCollectionAssociation.job_id == model.JobToOutputDatasetCollectionAssociation.job_id, + ) + .where(model.JobToInputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) + ) + return hda_select, hdca_select + + def inputs_for_hda(self, input_hda_id): + input_hdas = ( + select([literal("HistoryDatasetAssociation").label("src"), model.JobToInputDatasetAssociation.dataset_id.label("id")]) + .join( + model.JobToOutputDatasetAssociation, + model.JobToOutputDatasetAssociation.job_id == model.JobToInputDatasetAssociation.job_id, + ) + .where(model.JobToOutputDatasetAssociation.dataset_id == input_hda_id) + ) + input_hdcas = ( + select( + [ + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), + ] + ) + .join( + model.JobToOutputDatasetAssociation, + model.JobToOutputDatasetAssociation.job_id == model.JobToInputDatasetCollectionAssociation.job_id, + ) + .where(model.JobToOutputDatasetAssociation.dataset_id == input_hda_id) + ) + return input_hdas, input_hdcas + + def inputs_for_hdca(self, input_hdca_id): + input_hdas = ( + select([literal("HistoryDatasetAssociation").label("src"), model.JobToInputDatasetAssociation.dataset_id.label("id")]) + .join( + model.JobToOutputDatasetCollectionAssociation, + model.JobToOutputDatasetCollectionAssociation.job_id == model.JobToInputDatasetAssociation.job_id, + ) + .where(model.JobToOutputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) + ) + input_hdcas = ( + select( + [ + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), + ] + ) + .join( + model.JobToOutputDatasetCollectionAssociation, + model.JobToOutputDatasetCollectionAssociation.job_id == model.JobToInputDatasetCollectionAssociation.job_id, + ) + .where(model.JobToOutputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) + ) + return input_hdas, input_hdcas diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index dbbf09b269e..210e9e5895a 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -12,6 +12,10 @@ from typing import ( Union, ) +from sqlalchemy.sql import ( + select, + expression, +) from celery import group from pydantic import ( Extra, @@ -51,6 +55,7 @@ from galaxy.managers.history_contents import ( from galaxy.managers.jobs import ( fetch_job_states, summarize_jobs_to_dict, + JobConnectionsManager, ) from galaxy.managers.library_datasets import LibraryDatasetsManager from galaxy.model import ( @@ -939,6 +944,18 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto Allows additional filtering of contents and custom serialization. """ history = self._get_history(trans, history_id) + + # Related filter is included, get list of related item hids from jobs manager + if 'related-eq' in filter_query_params.q: + related_qv_index = filter_query_params.q.index('related-eq') + related_hid = filter_query_params.qv[related_qv_index] + found_item_data = trans.sa_session.execute(select(HistoryDatasetAssociation.id, expression.literal("HDA")).filter_by(history_id=history.id, hid=related_hid).union(select(HistoryDatasetCollectionAssociation.id, expression.literal("HDCA")).filter_by(history_id=history.id, hid=related_hid))).all() + job_connections_manager = JobConnectionsManager(trans.sa_session) + related = [] + for item_data in found_item_data: + related = job_connections_manager.get_related_hids(trans, item_data, related_hid) + filter_query_params.qv[related_qv_index] = related + filters = self.history_contents_filters.parse_query_filters(filter_query_params) stats_requested = accept == HistoryContentsWithStatsResult.__accept_type__ From 7d1176a51a84c55ce436dc03200e2c264172cc95 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Mon, 19 Dec 2022 19:35:04 -0500 Subject: [PATCH 02/15] Filter Inputs/Outputs: Create graph of id and src --- .../store/historyStore/historyItemsStore.js | 100 --------------- lib/galaxy/managers/jobs.py | 67 ++++++---- .../galaxy/services/history_contents.py | 9 +- .../managers/test_JobConnectionsManager.py | 114 ++++++++++++++++++ 4 files changed, 161 insertions(+), 129 deletions(-) delete mode 100644 client/src/store/historyStore/historyItemsStore.js create mode 100644 test/unit/app/managers/test_JobConnectionsManager.py diff --git a/client/src/store/historyStore/historyItemsStore.js b/client/src/store/historyStore/historyItemsStore.js deleted file mode 100644 index cee4b557d79..00000000000 --- a/client/src/store/historyStore/historyItemsStore.js +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Requests history items by reacting to changes of filter props passed - * to the history items provider used in the history panel. - */ - -import Vue from "vue"; -import { reverse } from "lodash"; -import { LastQueue } from "utils/promise-queue"; -import { urlData } from "utils/url"; -import { mergeArray } from "./model/utilities"; -import { getFilters, getQueryString, testFilters } from "utils/filterConversion"; - -const limit = 100; -const queue = new LastQueue(); - -const state = { - items: {}, - itemKey: "hid", - latestCreateTime: new Date(), - totalMatchesCount: undefined, - lastCheckedTime: new Date(), - matchedHids: {}, - isWatching: false, -}; - -const getters = { - getHistoryItems: - (state) => - ({ historyId, filterText }) => { - const itemArray = state.items[historyId] || []; - const filters = getFilters(filterText).filter((filter) => !filter.includes("related")); - const filtered = itemArray.filter((item) => { - if (!item) { - return false; - } - if (!testFilters(filters, item)) { - return false; - } - if (!state.matchedHids[historyId].includes(item.hid)) { - return false; - } - return true; - }); - return reverse(filtered); - }, - getLatestCreateTime: (state) => () => state.latestCreateTime, - getTotalMatchesCount: (state) => () => state.totalMatchesCount, - getLastCheckedTime: (state) => () => state.lastCheckedTime, - getWatchingVisibility: (state) => () => state.isWatching, -}; - -const actions = { - fetchHistoryItems: async ({ commit }, { historyId, filterText, offset }) => { - const queryString = getQueryString(filterText); - const params = `v=dev&order=hid&offset=${offset}&limit=${limit}`; - const url = `/api/histories/${historyId}/contents?${params}&${queryString}`; - const headers = { accept: "application/vnd.galaxy.history.contents.stats+json" }; - await queue.enqueue(urlData, { url, headers }, historyId).then((data) => { - const stats = data.stats; - commit("saveQueryStats", { stats }); - const payload = data.contents; - commit("saveHistoryItems", { historyId, payload }); - }); - }, -}; - -const mutations = { - saveHistoryItems: (state, { historyId, payload }) => { - // merges incoming payload into existing state - mergeArray(historyId, payload, state.items, state.itemKey); - const payloadHids = []; - // keep track of latest create time for items - payload.forEach((item) => { - if (item.state == "ok") { - const itemCreateTime = new Date(item.create_time); - if (itemCreateTime > state.latestCreateTime) { - state.latestCreateTime = itemCreateTime; - } - } - payloadHids.push(item.hid); - }); - Vue.set(state.matchedHids, historyId, payloadHids); - }, - setLastCheckedTime: (state, { checkForUpdate }) => { - state.lastCheckedTime = checkForUpdate; - }, - setWatchingVisibility: (state, { watchingVisibility }) => { - state.isWatching = watchingVisibility; - }, - saveQueryStats: (state, { stats }) => { - state.totalMatchesCount = stats.total_matches; - }, -}; - -export const historyItemsStore = { - state, - getters, - actions, - mutations, -}; diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index c9a9972dc95..a23b763d3af 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -16,10 +16,14 @@ from sqlalchemy import ( false, func, literal, + union, or_, ) from sqlalchemy.orm import aliased -from sqlalchemy.sql import select +from sqlalchemy.sql import ( + select, + expression, +) from galaxy import model from galaxy.exceptions import ( @@ -1019,31 +1023,52 @@ def summarize_job_outputs(job: model.Job, tool, params, security): class JobConnectionsManager: + """Get connections graph of inputs and outputs for given item""" + def __init__(self, sa_session: galaxy_scoped_session): self.sa_session = sa_session - def get_related_hids(self, trans, item_data, item_hid): - item_id = item_data[0] - item_type = item_data[1] - results = [item_hid] - - # Get select queries - if item_type == 'HDA': - output_selects = self.outputs_derived_from_input_hda(item_id) - input_selects = self.inputs_for_hda(item_id) - elif item_type == 'HDCA': - output_selects = self.outputs_derived_from_input_hdca(item_id) - input_selects = self.inputs_for_hdca(item_id) + def get_connections_graph(self, id, src): + """Get connections graph of inputs and outputs for given item id""" + if src == 'HistoryDatasetAssociation': + output_selects = self.outputs_derived_from_input_hda(id) + input_selects = self.inputs_for_hda(id) + elif src == 'HistoryDatasetCollectionAssociation': + output_selects = self.outputs_derived_from_input_hdca(id) + input_selects = self.inputs_for_hdca(id) else: - raise Exception(f"Invalid item type {item_type}") + raise Exception(f"Invalid item type {src}") + # Execute selects and return graph of inputs and outputs + result = {} + result["outputs"] = self._get_union_results(*output_selects) + result["inputs"] = self._get_union_results(*input_selects) + return result - # Add found related items' hids to results list - for select_query in (output_selects + input_selects): - query_res = self.sa_session.execute(select_query).all() - for res in query_res: - res_item = get_object(trans, res.id, res.src) - results.append(res_item.hid) - return results + def get_related_hids(self, trans, history_id, hid): + """Get connections graph of inputs and outputs for given item hid from the given history_id""" + # Get id(s) and src(s) for the given hid + items_by_hid = trans.sa_session.execute( + select(model.HistoryDatasetAssociation.id, expression.literal("HistoryDatasetAssociation")) + .filter_by(history_id=history_id, hid=hid) + .union( + select(model.HistoryDatasetCollectionAssociation.id, expression.literal("HistoryDatasetCollectionAssociation")) + .filter_by(history_id=history_id, hid=hid) + ) + ).all() + result = [hid] + for item_data in items_by_hid: + graph = self.get_connections_graph(id=item_data[0], src=item_data[1]) + # Add found related items' hids to result list + for val in (graph["outputs"] + graph["inputs"]): + item = get_object(trans, val["id"], val["src"]) + result.append(item.hid) + return result + + def _get_union_results(self, *selects): + result = [] + for row in self.sa_session.execute(union(*selects)).all(): + result.append({"src": row.src, "id": row.id}) + return result def outputs_derived_from_input_hda(self, input_hda_id): hda_select = ( diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 210e9e5895a..627aa4db402 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -12,10 +12,6 @@ from typing import ( Union, ) -from sqlalchemy.sql import ( - select, - expression, -) from celery import group from pydantic import ( Extra, @@ -949,11 +945,8 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto if 'related-eq' in filter_query_params.q: related_qv_index = filter_query_params.q.index('related-eq') related_hid = filter_query_params.qv[related_qv_index] - found_item_data = trans.sa_session.execute(select(HistoryDatasetAssociation.id, expression.literal("HDA")).filter_by(history_id=history.id, hid=related_hid).union(select(HistoryDatasetCollectionAssociation.id, expression.literal("HDCA")).filter_by(history_id=history.id, hid=related_hid))).all() job_connections_manager = JobConnectionsManager(trans.sa_session) - related = [] - for item_data in found_item_data: - related = job_connections_manager.get_related_hids(trans, item_data, related_hid) + related = job_connections_manager.get_related_hids(trans, history_id, related_hid) filter_query_params.qv[related_qv_index] = related filters = self.history_contents_filters.parse_query_filters(filter_query_params) diff --git a/test/unit/app/managers/test_JobConnectionsManager.py b/test/unit/app/managers/test_JobConnectionsManager.py new file mode 100644 index 00000000000..0b97a460eb8 --- /dev/null +++ b/test/unit/app/managers/test_JobConnectionsManager.py @@ -0,0 +1,114 @@ +import pytest +from sqlalchemy import union + +from galaxy.managers.jobs import JobConnectionsManager +from galaxy.model import ( + HistoryDatasetAssociation, + HistoryDatasetCollectionAssociation, + Job, +) +from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.model.unittest_utils import GalaxyDataTestApp + + +@pytest.fixture +def sa_session(): + return GalaxyDataTestApp().model.session + + +@pytest.fixture +def job_connections_manager(sa_session) -> JobConnectionsManager: + gm = JobConnectionsManager(sa_session) + return gm + + +# ============================================================================= +def setup_connected_dataset(sa_session: galaxy_scoped_session): + center_hda = HistoryDatasetAssociation(sa_session=sa_session) + input_hda = HistoryDatasetAssociation(sa_session=sa_session) + input_hdca = HistoryDatasetCollectionAssociation() + output_hda = HistoryDatasetAssociation(sa_session=sa_session) + output_hdca = HistoryDatasetCollectionAssociation() + input_job = Job() + output_job = Job() + input_job.add_output_dataset("output_hda", center_hda) + input_job.add_input_dataset("input_hda", input_hda) + input_job.add_input_dataset_collection("input_hdca", input_hdca) + output_job.add_input_dataset("input_hda", center_hda) + output_job.add_output_dataset("output_hda", output_hda) + output_job.add_output_dataset_collection("output_hdca", output_hdca) + sa_session.add_all([center_hda, input_hda, input_hdca, output_hdca, input_job, output_job]) + sa_session.flush() + expected_graph = { + "inputs": [{"src": "HistoryDatasetAssociation", "id": input_hda.id}, {"src": "HistoryDatasetCollectionAssociation", "id": input_hdca.id}], + "outputs": [{"src": "HistoryDatasetAssociation", "id": output_hda.id}, {"src": "HistoryDatasetCollectionAssociation", "id": output_hdca.id}], + } + return center_hda, expected_graph + + +def setup_connected_dataset_collection(sa_session: galaxy_scoped_session): + center_hdca = HistoryDatasetCollectionAssociation() + input_hda1 = HistoryDatasetAssociation(sa_session=sa_session) + input_hda2 = HistoryDatasetAssociation(sa_session=sa_session) + input_hdca = HistoryDatasetCollectionAssociation() + output_hda = HistoryDatasetAssociation(sa_session=sa_session) + output_hdca = HistoryDatasetCollectionAssociation() + input_job = Job() + output_job = Job() + input_job.add_input_dataset_collection("output_hdca", center_hdca) + input_job.add_input_dataset("input_hda", input_hda1) + input_job.add_input_dataset("input_hda", input_hda2) + input_job.add_input_dataset_collection("input_hdca", input_hdca) + output_job.add_input_dataset_collection("input_hdca", center_hdca) + output_job.add_output_dataset("output_hda", output_hda) + output_job.add_output_dataset_collection("output_hdca", output_hdca) + sa_session.add_all([center_hdca, input_hda1, input_hda2, input_hdca, output_hdca, input_job, output_job]) + sa_session.flush() + expected_graph = { + "inputs": [ + {"src": "HistoryDatasetAssociation", "id": input_hda1.id}, + {"src": "HistoryDatasetAssociation", "id": input_hda2.id}, + {"src": "HistoryDatasetCollectionAssociation", "id": input_hdca.id} + ], + "outputs": [{"src": "HistoryDatasetAssociation", "id": output_hda.id}, {"src": "HistoryDatasetCollectionAssociation", "id": output_hdca.id}], + } + return center_hdca, expected_graph + + +# ============================================================================= +def test_graph_manager_inputs_for_hda(job_connections_manager: JobConnectionsManager): + sa_session = job_connections_manager.sa_session + center_hda, expected_graph = setup_connected_dataset(sa_session) + s = job_connections_manager.inputs_for_hda(center_hda.id) + assert len(sa_session.execute(union(*s)).all()) == 2 + + +def test_graph_manager_outputs_for_hda(job_connections_manager: JobConnectionsManager): + sa_session = job_connections_manager.sa_session + center_hda, expected_graph = setup_connected_dataset(sa_session) + s = job_connections_manager.outputs_derived_from_input_hda(center_hda.id) + assert len(sa_session.execute(union(*s)).all()) == 2 + + +def test_graph_manager_inputs_for_hdca(job_connections_manager: JobConnectionsManager): + sa_session = job_connections_manager.sa_session + center_hdca, expected_graph = setup_connected_dataset_collection(sa_session) + s = job_connections_manager.inputs_for_hdca(center_hdca.id) + assert len(sa_session.execute(union(*s)).all()) == 2 + + +def test_graph_manager_outputs_for_hdca(job_connections_manager: JobConnectionsManager): + sa_session = job_connections_manager.sa_session + center_hdca, expected_graph = setup_connected_dataset_collection(sa_session) + s = job_connections_manager.outputs_derived_from_input_hdca(center_hdca.id) + assert len(sa_session.execute(union(*s)).all()) == 2 + + +def test_graph_manager_hda(job_connections_manager: JobConnectionsManager): + center_hda, expected_graph = setup_connected_dataset(job_connections_manager.sa_session) + assert job_connections_manager.get_connections_graph(center_hda.id, "HistoryDatasetAssociation") == expected_graph + + +def test_graph_manager_hdca(job_connections_manager: JobConnectionsManager): + center_hdca, expected_graph = setup_connected_dataset_collection(job_connections_manager.sa_session) + assert job_connections_manager.get_connections_graph(center_hdca.id, "HistoryDatasetCollectionAssociation") == expected_graph From 26609e52111c1ee76cdc142bce46cd92e1efb64e Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Mon, 19 Dec 2022 23:59:38 -0500 Subject: [PATCH 03/15] Filter Inputs and Outputs: Improve store related hids storage --- .../History/Content/ContentItem.vue | 2 +- .../History/CurrentHistory/HistoryPanel.vue | 2 +- .../src/stores/history/historyItemsStore.js | 19 ++++++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/client/src/components/History/Content/ContentItem.vue b/client/src/components/History/Content/ContentItem.vue index 0f93ee627d6..4ee8f845d04 100644 --- a/client/src/components/History/Content/ContentItem.vue +++ b/client/src/components/History/Content/ContentItem.vue @@ -18,7 +18,7 @@ this.highlightsKey) { return "output"; diff --git a/client/src/stores/history/historyItemsStore.js b/client/src/stores/history/historyItemsStore.js index fcdbe0a2409..2f4659b939c 100644 --- a/client/src/stores/history/historyItemsStore.js +++ b/client/src/stores/history/historyItemsStore.js @@ -21,7 +21,7 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { latestCreateTime: new Date(), totalMatchesCount: undefined, lastCheckedTime: new Date(), - matchedHids: {}, + relatedItems: {}, isWatching: false, }), getters: { @@ -29,6 +29,7 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { return (historyId, filterText) => { const itemArray = state.items[historyId] || []; const filters = HistoryFilters.getFilters(filterText).filter((filter) => !filter.includes("related")); + const relatedHid = HistoryFilters.getQueryDict(filterText)["related-eq"] || null; const filtered = itemArray.filter((item) => { if (!item) { return false; @@ -36,7 +37,8 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { if (!HistoryFilters.testFilters(filters, item)) { return false; } - if (!state.matchedHids[historyId].includes(item.hid)) { + const relationKey = `${historyId}-${relatedHid}-${item.hid}`; + if (relatedHid && !state.relatedItems[relationKey]) { return false; } return true; @@ -60,6 +62,8 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { actions: { async fetchHistoryItems(historyId, filterText, offset) { const queryString = HistoryFilters.getQueryString(filterText); + const filters = HistoryFilters.getQueryDict(filterText); + const relatedHid = filters["related-eq"] || null; const params = `v=dev&order=hid&offset=${offset}&limit=${limit}`; const url = `/api/histories/${historyId}/contents?${params}&${queryString}`; const headers = { accept: "application/vnd.galaxy.history.contents.stats+json" }; @@ -67,15 +71,14 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { const stats = data.stats; this.totalMatchesCount = stats.total_matches; const payload = data.contents; - this.saveHistoryItems(historyId, payload); + this.saveHistoryItems(historyId, payload, relatedHid); }); }, // Setters - saveHistoryItems(historyId, payload) { + saveHistoryItems(historyId, payload, relatedHid = null) { this.$patch((state) => { // merges incoming payload into existing state mergeArray(historyId, payload, state.items, state.itemKey); - const payloadHids = []; // keep track of latest create time for items payload.forEach((item) => { if (item.state == "ok") { @@ -84,9 +87,11 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { state.latestCreateTime = itemCreateTime; } } - payloadHids.push(item.hid); + if (relatedHid) { + const relationKey = `${historyId}-${relatedHid}-${item.hid}`; + Vue.set(state.relatedItems, relationKey, true); + } }); - Vue.set(state.matchedHids, historyId, payloadHids); }); }, setLastCheckedTime(checkForUpdate) { From 66c16c8a2b21310f83f73226dcdabef65a06ec91 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Tue, 20 Dec 2022 20:15:40 -0500 Subject: [PATCH 04/15] Create getFilterValue function that gets value of filter from filterText --- .../src/stores/history/historyItemsStore.js | 5 +- client/src/utils/filtering.test.js | 11 ++++ client/src/utils/filtering.ts | 54 +++++++++++++++++-- 3 files changed, 64 insertions(+), 6 deletions(-) diff --git a/client/src/stores/history/historyItemsStore.js b/client/src/stores/history/historyItemsStore.js index 2f4659b939c..7a0563dc297 100644 --- a/client/src/stores/history/historyItemsStore.js +++ b/client/src/stores/history/historyItemsStore.js @@ -29,7 +29,7 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { return (historyId, filterText) => { const itemArray = state.items[historyId] || []; const filters = HistoryFilters.getFilters(filterText).filter((filter) => !filter.includes("related")); - const relatedHid = HistoryFilters.getQueryDict(filterText)["related-eq"] || null; + const relatedHid = HistoryFilters.getFilterValue(filterText, "related"); const filtered = itemArray.filter((item) => { if (!item) { return false; @@ -62,8 +62,6 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { actions: { async fetchHistoryItems(historyId, filterText, offset) { const queryString = HistoryFilters.getQueryString(filterText); - const filters = HistoryFilters.getQueryDict(filterText); - const relatedHid = filters["related-eq"] || null; const params = `v=dev&order=hid&offset=${offset}&limit=${limit}`; const url = `/api/histories/${historyId}/contents?${params}&${queryString}`; const headers = { accept: "application/vnd.galaxy.history.contents.stats+json" }; @@ -71,6 +69,7 @@ export const useHistoryItemsStore = defineStore("historyItemsStore", { const stats = data.stats; this.totalMatchesCount = stats.total_matches; const payload = data.contents; + const relatedHid = HistoryFilters.getFilterValue(filterText, "related"); this.saveHistoryItems(historyId, payload, relatedHid); }); }, diff --git a/client/src/utils/filtering.test.js b/client/src/utils/filtering.test.js index b7f877a9ab0..da88b73a243 100644 --- a/client/src/utils/filtering.test.js +++ b/client/src/utils/filtering.test.js @@ -29,12 +29,23 @@ describe("filtering", () => { expect(queryDict["name-contains"]).toBe("name of item"); }); test("parse check filter", () => { + expect(HistoryFilters.checkFilter(filterTexts[0], "name", "name of item")).toBe(true); expect(HistoryFilters.checkFilter(filterTexts[0], "tag", "first")).toBe(true); expect(HistoryFilters.checkFilter(filterTexts[0], "tag", "second")).toBe(false); expect(HistoryFilters.checkFilter(filterTexts[0], "deleted", "false")).toBe(true); expect(HistoryFilters.checkFilter(filterTexts[0], "visible", true)).toBe(true); expect(HistoryFilters.checkFilter(filterTexts[0], "visible", "false")).toBe(false); }); + test("parse get filter value", () => { + expect(HistoryFilters.getFilterValue(filterTexts[0], "name")).toBe("name of item"); + expect(HistoryFilters.getFilterValue(filterTexts[0], "hid", "gt")).toBe("10"); + expect(HistoryFilters.getFilterValue(filterTexts[0], "hid", "lt")).toBe("100"); + expect(HistoryFilters.getFilterValue(filterTexts[0], "tag")).toBe("first"); + expect(HistoryFilters.getFilterValue(filterTexts[0], "deleted")).toBe("False"); + expect(HistoryFilters.getFilterValue(filterTexts[0], "visible")).toBe("TRUE"); + expect(HistoryFilters.getFilterValue(filterTexts[1], "hid", "gt")).toBe("10"); + expect(HistoryFilters.getFilterValue(filterTexts[1], "create_time", "gt")).toBe("2021-01-01"); + }); test("parse filter text as entries", () => { filterTexts.forEach((filterText) => { const filters = HistoryFilters.getFilters(filterText); diff --git a/client/src/utils/filtering.ts b/client/src/utils/filtering.ts index 616bd3a3858..9c472e55e10 100644 --- a/client/src/utils/filtering.ts +++ b/client/src/utils/filtering.ts @@ -61,6 +61,27 @@ export function expandNameTag(value: string | object): string { return toLower(value); } +/** Converts string alias to string operator, e.g.: 'gt' to '>' + * @param {string} alias + * @returns {string} Arithmetic operator, e.g.: '>' + * */ +export function getOperatorForAlias(alias: string): string { + switch (alias) { + case "lt": + return "<"; + case "le": + return "<="; + case "ge": + return ">="; + case "gt": + return ">"; + case "eq": + return ":"; + default: + return ""; + } +} + type HandlerReturn = { attribute: string; converter?: Converter; @@ -323,12 +344,39 @@ export default class Filtering { * @returns {Boolean} True if the filter is set to the given value * */ checkFilter(filterText: string, filterName: string, filterValue: T): boolean { - const re = new RegExp(`${filterName}:(\\S+)`); - const reMatch = re.exec(filterText); - const testValue = reMatch ? reMatch[1] : this.defaultFilters[filterName]; + const testValue = this.getFilterValue(filterText, filterName); return toLowerNoQuotes(testValue) === toLowerNoQuotes(filterValue); } + /** Get the value of a particular filter from filterText. + * @param {String} filterText Raw filter text string + * @param {String} filterName Filter key to check + * @returns {String | Boolean} The filterValue for the filter + * */ + getFilterValue(filterText: string, filterName: string, alias = "eq"): string | boolean { + const op = getOperatorForAlias(alias); + const reInQuotes = `'([^']*[^\\s']*)'`; + const reNoQuotes = `(\\S+)`; + // Array of re groups, note: order matters as in quote filterVals must be checked first + const reGroups = [ + `${filterName}${op}${reInQuotes}`, + `${filterName}${op}${reNoQuotes}`, + `${filterName}-${alias}:${reInQuotes}`, + `${filterName}-${alias}:${reNoQuotes}`, + `${filterName}_${alias}:${reInQuotes}`, + `${filterName}_${alias}:${reNoQuotes}`, + ]; + let reMatch = null; + for (const reString of reGroups) { + const re = new RegExp(reString); + reMatch = re.exec(filterText); + if (reMatch) { + break; + } + } + return reMatch ? reMatch[1] : this.defaultFilters[filterName]; + } + /** Test if an item passes all filters. * @param {Object} filters Parsed in key-value pairs from getFilters() * @param {Object} item Item to test against the filters From 3554bf10e1b0ca127784e14917b788d5079d4a21 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Tue, 20 Dec 2022 22:29:53 -0500 Subject: [PATCH 05/15] Use better typing in filtering.ts --- client/src/utils/filtering.ts | 40 +++++++++++++++-------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/client/src/utils/filtering.ts b/client/src/utils/filtering.ts index 9c472e55e10..f4b8c2b1430 100644 --- a/client/src/utils/filtering.ts +++ b/client/src/utils/filtering.ts @@ -18,6 +18,14 @@ const defaultValidAliases = [ ["<", "_lt"], ]; +const operatorForAlias: Record = { + lt: "<", + le: "<=", + ge: ">=", + gt: ">", + eq: ":", +}; + /** Converts user input to backend compatible date * @param {string} value * @returns {Number} seconds since epoch @@ -66,20 +74,7 @@ export function expandNameTag(value: string | object): string { * @returns {string} Arithmetic operator, e.g.: '>' * */ export function getOperatorForAlias(alias: string): string { - switch (alias) { - case "lt": - return "<"; - case "le": - return "<="; - case "ge": - return ">="; - case "gt": - return ">"; - case "eq": - return ":"; - default: - return ""; - } + return operatorForAlias[alias]; } type HandlerReturn = { @@ -357,7 +352,7 @@ export default class Filtering { const op = getOperatorForAlias(alias); const reInQuotes = `'([^']*[^\\s']*)'`; const reNoQuotes = `(\\S+)`; - // Array of re groups, note: order matters as in quote filterVals must be checked first + // Array of re groups, note: order matters as in-quote filterVals must be checked first const reGroups = [ `${filterName}${op}${reInQuotes}`, `${filterName}${op}${reNoQuotes}`, @@ -366,15 +361,14 @@ export default class Filtering { `${filterName}_${alias}:${reInQuotes}`, `${filterName}_${alias}:${reNoQuotes}`, ]; - let reMatch = null; - for (const reString of reGroups) { - const re = new RegExp(reString); - reMatch = re.exec(filterText); - if (reMatch) { - break; - } + const reString = reGroups.join("|"); + const re = new RegExp(reString); + const reMatch = re.exec(filterText); + let filterVal = null; + if (reMatch) { + filterVal = reMatch.slice(1, reGroups.length + 1).find((val) => val); } - return reMatch ? reMatch[1] : this.defaultFilters[filterName]; + return filterVal || this.defaultFilters[filterName]; } /** Test if an item passes all filters. From d7372f61fc169a25b748b1e64bb82b7824ef8b1d Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Wed, 21 Dec 2022 14:35:18 -0500 Subject: [PATCH 06/15] Improve regex in filtering.ts --- client/src/utils/filtering.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/client/src/utils/filtering.ts b/client/src/utils/filtering.ts index f4b8c2b1430..9d83d77bf66 100644 --- a/client/src/utils/filtering.ts +++ b/client/src/utils/filtering.ts @@ -350,23 +350,12 @@ export default class Filtering { * */ getFilterValue(filterText: string, filterName: string, alias = "eq"): string | boolean { const op = getOperatorForAlias(alias); - const reInQuotes = `'([^']*[^\\s']*)'`; - const reNoQuotes = `(\\S+)`; - // Array of re groups, note: order matters as in-quote filterVals must be checked first - const reGroups = [ - `${filterName}${op}${reInQuotes}`, - `${filterName}${op}${reNoQuotes}`, - `${filterName}-${alias}:${reInQuotes}`, - `${filterName}-${alias}:${reNoQuotes}`, - `${filterName}_${alias}:${reInQuotes}`, - `${filterName}_${alias}:${reNoQuotes}`, - ]; - const reString = reGroups.join("|"); + const reString = `${filterName}(?:${op}|[-|_]${alias}:)(?:'([^']*[^\\s']*)'|(\\S+))`; const re = new RegExp(reString); const reMatch = re.exec(filterText); let filterVal = null; if (reMatch) { - filterVal = reMatch.slice(1, reGroups.length + 1).find((val) => val); + filterVal = reMatch[1] || reMatch[2]; } return filterVal || this.defaultFilters[filterName]; } From 06f28f32883f235428986479b996da75684f7338 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Fri, 23 Dec 2022 12:34:03 -0500 Subject: [PATCH 07/15] Filter Inputs/Outputs: Fix backend test, fix mypy issues --- client/src/utils/filtering.ts | 1 + lib/galaxy/webapps/galaxy/services/history_contents.py | 5 ++++- test/unit/app/managers/test_JobConnectionsManager.py | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/client/src/utils/filtering.ts b/client/src/utils/filtering.ts index 9d83d77bf66..1c525ca9b20 100644 --- a/client/src/utils/filtering.ts +++ b/client/src/utils/filtering.ts @@ -346,6 +346,7 @@ export default class Filtering { /** Get the value of a particular filter from filterText. * @param {String} filterText Raw filter text string * @param {String} filterName Filter key to check + * @param {String} [alias="eq"] String alias for filter operator, e.g.:"lt" * @returns {String | Boolean} The filterValue for the filter * */ getFilterValue(filterText: string, filterName: string, alias = "eq"): string | boolean { diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 627aa4db402..54a39dd8666 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -941,8 +941,11 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto """ history = self._get_history(trans, history_id) + # TODO: Would be better if the JobConnections Manager could be used in + # history_contents manager instead, can't now because of circular import there + # Related filter is included, get list of related item hids from jobs manager - if 'related-eq' in filter_query_params.q: + if filter_query_params.q and filter_query_params.qv and 'related-eq' in filter_query_params.q: related_qv_index = filter_query_params.q.index('related-eq') related_hid = filter_query_params.qv[related_qv_index] job_connections_manager = JobConnectionsManager(trans.sa_session) diff --git a/test/unit/app/managers/test_JobConnectionsManager.py b/test/unit/app/managers/test_JobConnectionsManager.py index 0b97a460eb8..e63a9645519 100644 --- a/test/unit/app/managers/test_JobConnectionsManager.py +++ b/test/unit/app/managers/test_JobConnectionsManager.py @@ -55,7 +55,7 @@ def setup_connected_dataset_collection(sa_session: galaxy_scoped_session): output_hdca = HistoryDatasetCollectionAssociation() input_job = Job() output_job = Job() - input_job.add_input_dataset_collection("output_hdca", center_hdca) + input_job.add_output_dataset_collection("output_hdca", center_hdca) input_job.add_input_dataset("input_hda", input_hda1) input_job.add_input_dataset("input_hda", input_hda2) input_job.add_input_dataset_collection("input_hdca", input_hdca) @@ -94,7 +94,7 @@ def test_graph_manager_inputs_for_hdca(job_connections_manager: JobConnectionsMa sa_session = job_connections_manager.sa_session center_hdca, expected_graph = setup_connected_dataset_collection(sa_session) s = job_connections_manager.inputs_for_hdca(center_hdca.id) - assert len(sa_session.execute(union(*s)).all()) == 2 + assert len(sa_session.execute(union(*s)).all()) == 3 def test_graph_manager_outputs_for_hdca(job_connections_manager: JobConnectionsManager): From 0cd4cef9c269ab958220dca2b5b512c63981dfcc Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Thu, 12 Jan 2023 23:05:03 +0500 Subject: [PATCH 08/15] Update client/src/utils/filtering.ts with typing fixes --- .../History/Content/ContentItem.vue | 24 ++++++++++++------- client/src/utils/filtering.ts | 12 +++++----- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/client/src/components/History/Content/ContentItem.vue b/client/src/components/History/Content/ContentItem.vue index 4ee8f845d04..34171c74de5 100644 --- a/client/src/components/History/Content/ContentItem.vue +++ b/client/src/components/History/Content/ContentItem.vue @@ -14,26 +14,34 @@ - + - - + - - + - + diff --git a/client/src/utils/filtering.ts b/client/src/utils/filtering.ts index 1c525ca9b20..5ea02047fb8 100644 --- a/client/src/utils/filtering.ts +++ b/client/src/utils/filtering.ts @@ -70,8 +70,8 @@ export function expandNameTag(value: string | object): string { } /** Converts string alias to string operator, e.g.: 'gt' to '>' - * @param {string} alias - * @returns {string} Arithmetic operator, e.g.: '>' + * @param alias + * @returns Arithmetic operator, e.g.: '>' * */ export function getOperatorForAlias(alias: string): string { return operatorForAlias[alias]; @@ -344,10 +344,10 @@ export default class Filtering { } /** Get the value of a particular filter from filterText. - * @param {String} filterText Raw filter text string - * @param {String} filterName Filter key to check - * @param {String} [alias="eq"] String alias for filter operator, e.g.:"lt" - * @returns {String | Boolean} The filterValue for the filter + * @param filterText Raw filter text string + * @param filterName Filter key to check + * @param [alias="eq"] String alias for filter operator, e.g.:"lt" + * @returns The filterValue for the filter * */ getFilterValue(filterText: string, filterName: string, alias = "eq"): string | boolean { const op = getOperatorForAlias(alias); From 766515696cd8762e36a7f84fc59568c48c16f785 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Sun, 15 Jan 2023 18:32:26 +0500 Subject: [PATCH 09/15] Move JobConnections manager to its own file --- lib/galaxy/managers/history_contents.py | 28 ++++ lib/galaxy/managers/job_connections.py | 158 ++++++++++++++++++ lib/galaxy/managers/jobs.py | 153 +---------------- .../galaxy/services/history_contents.py | 14 +- .../managers/test_JobConnectionsManager.py | 2 +- 5 files changed, 189 insertions(+), 166 deletions(-) create mode 100644 lib/galaxy/managers/job_connections.py diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 6d2d6314d63..3be17f5a0a3 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -32,6 +32,7 @@ from galaxy import ( exceptions as glx_exceptions, model, ) +from galaxy.schema import ValueFilterQueryParams from galaxy.managers import ( annotatable, base, @@ -41,6 +42,7 @@ from galaxy.managers import ( taggable, tools, ) +from galaxy.managers.job_connections import JobConnectionsManager from galaxy.structured_app import MinimalManagerApp from .base import ( parse_bool, @@ -525,6 +527,30 @@ class HistoryContentsFilters( # surprisingly (but ominously), this works for both content classes in the union that's filtered model_class = model.HistoryDatasetAssociation + def parse_query_filters_with_relations(self, query_filters: ValueFilterQueryParams, history_id): + """Parse query filters but consider case where related filter is included.""" + if query_filters.q and query_filters.qv and 'related-eq' in query_filters.q: + qv_index = query_filters.q.index('related-eq') + qv_hid = query_filters.qv[qv_index] + + # Make new q and qv excluding related filter + new_q = [x for i, x in enumerate(query_filters.q) if i != qv_index] + new_qv = [x for i, x in enumerate(query_filters.qv) if i != qv_index] + + # Get list of related item hids from job_connections manager + job_connections_manager = JobConnectionsManager(self.app.model.session) + related = job_connections_manager.get_related_hids(history_id, qv_hid) + + # Make new query_filters with updated list of related hids for given hid + new_q.append("related-eq") + new_qv.append(str(related)) + query_filters_with_relations = ValueFilterQueryParams( + q=new_q, + qv=new_qv, + ) + return super().parse_query_filters(query_filters_with_relations) + return super().parse_query_filters(query_filters) + def _parse_orm_filter(self, attr, op, val): # we need to use some manual/text/column fu here since some where clauses on the union don't work @@ -539,6 +565,8 @@ class HistoryContentsFilters( raise_filter_err(attr, op, val, "bad op in filter") if attr == "related" and op == "eq": + if isinstance(val, str): + val = val.strip('][').split(', ') return sql.column("hid").in_(val) if attr == "type_id": diff --git a/lib/galaxy/managers/job_connections.py b/lib/galaxy/managers/job_connections.py new file mode 100644 index 00000000000..2a8254ab227 --- /dev/null +++ b/lib/galaxy/managers/job_connections.py @@ -0,0 +1,158 @@ +from sqlalchemy import ( + literal, + union, +) +from sqlalchemy.sql import ( + select, + expression, +) + +from galaxy import model +from galaxy.managers.base import get_class +from galaxy.model.scoped_session import galaxy_scoped_session + + +class JobConnectionsManager: + """Get connections graph of inputs and outputs for given item""" + + def __init__(self, sa_session: galaxy_scoped_session): + self.sa_session = sa_session + + def get_connections_graph(self, id, src): + """Get connections graph of inputs and outputs for given item id""" + if src == 'HistoryDatasetAssociation': + output_selects = self.outputs_derived_from_input_hda(id) + input_selects = self.inputs_for_hda(id) + elif src == 'HistoryDatasetCollectionAssociation': + output_selects = self.outputs_derived_from_input_hdca(id) + input_selects = self.inputs_for_hdca(id) + else: + raise Exception(f"Invalid item type {src}") + # Execute selects and return graph of inputs and outputs + result = {} + result["outputs"] = self._get_union_results(*output_selects) + result["inputs"] = self._get_union_results(*input_selects) + return result + + def get_related_hids(self, history_id, hid): + """Get connections graph of inputs and outputs for given item hid from the given history_id""" + # Get id(s) and src(s) for the given hid + items_by_hid = self.sa_session.execute( + select(model.HistoryDatasetAssociation.id, expression.literal("HistoryDatasetAssociation")) + .filter_by(history_id=history_id, hid=hid) + .union( + select(model.HistoryDatasetCollectionAssociation.id, expression.literal("HistoryDatasetCollectionAssociation")) + .filter_by(history_id=history_id, hid=hid) + ) + ).all() + result = [int(hid)] + for item_data in items_by_hid: + graph = self.get_connections_graph(id=item_data[0], src=item_data[1]) + # Add found related items' hids to result list + for val in (graph["outputs"] + graph["inputs"]): + item_class = get_class(val["src"]) + item = self.sa_session.query(item_class).get(val["id"]) + result.append(item.hid) + return result + + def _get_union_results(self, *selects): + result = [] + for row in self.sa_session.execute(union(*selects)).all(): + result.append({"src": row.src, "id": row.id}) + return result + + def outputs_derived_from_input_hda(self, input_hda_id): + hda_select = ( + select([literal("HistoryDatasetAssociation").label("src"), model.JobToOutputDatasetAssociation.dataset_id.label("id")]) + .join( + model.JobToInputDatasetAssociation, + model.JobToInputDatasetAssociation.job_id == model.JobToOutputDatasetAssociation.job_id, + ) + .where(model.JobToInputDatasetAssociation.dataset_id == input_hda_id) + ) + hdca_select = ( + select( + [ + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), + ] + ) + .join( + model.JobToInputDatasetAssociation, + model.JobToInputDatasetAssociation.job_id == model.JobToOutputDatasetCollectionAssociation.job_id, + ) + .where(model.JobToInputDatasetAssociation.dataset_id == input_hda_id) + ) + return hda_select, hdca_select + + def outputs_derived_from_input_hdca(self, input_hdca_id): + hda_select = ( + select([literal("HistoryDatasetAssociation").label("src"), model.JobToOutputDatasetAssociation.dataset_id.label("id")]) + .join( + model.JobToInputDatasetCollectionAssociation, + model.JobToInputDatasetCollectionAssociation.job_id == model.JobToOutputDatasetAssociation.job_id, + ) + .where(model.JobToInputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) + ) + hdca_select = ( + select( + [ + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), + ] + ) + .join( + model.JobToInputDatasetCollectionAssociation, + model.JobToInputDatasetCollectionAssociation.job_id == model.JobToOutputDatasetCollectionAssociation.job_id, + ) + .where(model.JobToInputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) + ) + return hda_select, hdca_select + + def inputs_for_hda(self, input_hda_id): + input_hdas = ( + select([literal("HistoryDatasetAssociation").label("src"), model.JobToInputDatasetAssociation.dataset_id.label("id")]) + .join( + model.JobToOutputDatasetAssociation, + model.JobToOutputDatasetAssociation.job_id == model.JobToInputDatasetAssociation.job_id, + ) + .where(model.JobToOutputDatasetAssociation.dataset_id == input_hda_id) + ) + input_hdcas = ( + select( + [ + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), + ] + ) + .join( + model.JobToOutputDatasetAssociation, + model.JobToOutputDatasetAssociation.job_id == model.JobToInputDatasetCollectionAssociation.job_id, + ) + .where(model.JobToOutputDatasetAssociation.dataset_id == input_hda_id) + ) + return input_hdas, input_hdcas + + def inputs_for_hdca(self, input_hdca_id): + input_hdas = ( + select([literal("HistoryDatasetAssociation").label("src"), model.JobToInputDatasetAssociation.dataset_id.label("id")]) + .join( + model.JobToOutputDatasetCollectionAssociation, + model.JobToOutputDatasetCollectionAssociation.job_id == model.JobToInputDatasetAssociation.job_id, + ) + .where(model.JobToOutputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) + ) + input_hdcas = ( + select( + [ + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), + ] + ) + .join( + model.JobToOutputDatasetCollectionAssociation, + model.JobToOutputDatasetCollectionAssociation.job_id == model.JobToInputDatasetCollectionAssociation.job_id, + ) + .where(model.JobToOutputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) + ) + return input_hdas, input_hdcas diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index a23b763d3af..a32a994b602 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -15,15 +15,10 @@ from sqlalchemy import ( and_, false, func, - literal, - union, or_, ) from sqlalchemy.orm import aliased -from sqlalchemy.sql import ( - select, - expression, -) +from sqlalchemy.sql import select from galaxy import model from galaxy.exceptions import ( @@ -36,7 +31,6 @@ from galaxy.job_metrics import ( RawMetric, Safety, ) -from galaxy.managers.base import get_object from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.datasets import DatasetManager from galaxy.managers.hdas import HDAManager @@ -1020,148 +1014,3 @@ def summarize_job_outputs(job: model.Job, tool, params, security): } ) return outputs - - -class JobConnectionsManager: - """Get connections graph of inputs and outputs for given item""" - - def __init__(self, sa_session: galaxy_scoped_session): - self.sa_session = sa_session - - def get_connections_graph(self, id, src): - """Get connections graph of inputs and outputs for given item id""" - if src == 'HistoryDatasetAssociation': - output_selects = self.outputs_derived_from_input_hda(id) - input_selects = self.inputs_for_hda(id) - elif src == 'HistoryDatasetCollectionAssociation': - output_selects = self.outputs_derived_from_input_hdca(id) - input_selects = self.inputs_for_hdca(id) - else: - raise Exception(f"Invalid item type {src}") - # Execute selects and return graph of inputs and outputs - result = {} - result["outputs"] = self._get_union_results(*output_selects) - result["inputs"] = self._get_union_results(*input_selects) - return result - - def get_related_hids(self, trans, history_id, hid): - """Get connections graph of inputs and outputs for given item hid from the given history_id""" - # Get id(s) and src(s) for the given hid - items_by_hid = trans.sa_session.execute( - select(model.HistoryDatasetAssociation.id, expression.literal("HistoryDatasetAssociation")) - .filter_by(history_id=history_id, hid=hid) - .union( - select(model.HistoryDatasetCollectionAssociation.id, expression.literal("HistoryDatasetCollectionAssociation")) - .filter_by(history_id=history_id, hid=hid) - ) - ).all() - result = [hid] - for item_data in items_by_hid: - graph = self.get_connections_graph(id=item_data[0], src=item_data[1]) - # Add found related items' hids to result list - for val in (graph["outputs"] + graph["inputs"]): - item = get_object(trans, val["id"], val["src"]) - result.append(item.hid) - return result - - def _get_union_results(self, *selects): - result = [] - for row in self.sa_session.execute(union(*selects)).all(): - result.append({"src": row.src, "id": row.id}) - return result - - def outputs_derived_from_input_hda(self, input_hda_id): - hda_select = ( - select([literal("HistoryDatasetAssociation").label("src"), model.JobToOutputDatasetAssociation.dataset_id.label("id")]) - .join( - model.JobToInputDatasetAssociation, - model.JobToInputDatasetAssociation.job_id == model.JobToOutputDatasetAssociation.job_id, - ) - .where(model.JobToInputDatasetAssociation.dataset_id == input_hda_id) - ) - hdca_select = ( - select( - [ - literal("HistoryDatasetCollectionAssociation").label("src"), - model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), - ] - ) - .join( - model.JobToInputDatasetAssociation, - model.JobToInputDatasetAssociation.job_id == model.JobToOutputDatasetCollectionAssociation.job_id, - ) - .where(model.JobToInputDatasetAssociation.dataset_id == input_hda_id) - ) - return hda_select, hdca_select - - def outputs_derived_from_input_hdca(self, input_hdca_id): - hda_select = ( - select([literal("HistoryDatasetAssociation").label("src"), model.JobToOutputDatasetAssociation.dataset_id.label("id")]) - .join( - model.JobToInputDatasetCollectionAssociation, - model.JobToInputDatasetCollectionAssociation.job_id == model.JobToOutputDatasetAssociation.job_id, - ) - .where(model.JobToInputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) - ) - hdca_select = ( - select( - [ - literal("HistoryDatasetCollectionAssociation").label("src"), - model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), - ] - ) - .join( - model.JobToInputDatasetCollectionAssociation, - model.JobToInputDatasetCollectionAssociation.job_id == model.JobToOutputDatasetCollectionAssociation.job_id, - ) - .where(model.JobToInputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) - ) - return hda_select, hdca_select - - def inputs_for_hda(self, input_hda_id): - input_hdas = ( - select([literal("HistoryDatasetAssociation").label("src"), model.JobToInputDatasetAssociation.dataset_id.label("id")]) - .join( - model.JobToOutputDatasetAssociation, - model.JobToOutputDatasetAssociation.job_id == model.JobToInputDatasetAssociation.job_id, - ) - .where(model.JobToOutputDatasetAssociation.dataset_id == input_hda_id) - ) - input_hdcas = ( - select( - [ - literal("HistoryDatasetCollectionAssociation").label("src"), - model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), - ] - ) - .join( - model.JobToOutputDatasetAssociation, - model.JobToOutputDatasetAssociation.job_id == model.JobToInputDatasetCollectionAssociation.job_id, - ) - .where(model.JobToOutputDatasetAssociation.dataset_id == input_hda_id) - ) - return input_hdas, input_hdcas - - def inputs_for_hdca(self, input_hdca_id): - input_hdas = ( - select([literal("HistoryDatasetAssociation").label("src"), model.JobToInputDatasetAssociation.dataset_id.label("id")]) - .join( - model.JobToOutputDatasetCollectionAssociation, - model.JobToOutputDatasetCollectionAssociation.job_id == model.JobToInputDatasetAssociation.job_id, - ) - .where(model.JobToOutputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) - ) - input_hdcas = ( - select( - [ - literal("HistoryDatasetCollectionAssociation").label("src"), - model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), - ] - ) - .join( - model.JobToOutputDatasetCollectionAssociation, - model.JobToOutputDatasetCollectionAssociation.job_id == model.JobToInputDatasetCollectionAssociation.job_id, - ) - .where(model.JobToOutputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) - ) - return input_hdas, input_hdcas diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 54a39dd8666..0c72c4b497c 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -51,7 +51,6 @@ from galaxy.managers.history_contents import ( from galaxy.managers.jobs import ( fetch_job_states, summarize_jobs_to_dict, - JobConnectionsManager, ) from galaxy.managers.library_datasets import LibraryDatasetsManager from galaxy.model import ( @@ -941,18 +940,7 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto """ history = self._get_history(trans, history_id) - # TODO: Would be better if the JobConnections Manager could be used in - # history_contents manager instead, can't now because of circular import there - - # Related filter is included, get list of related item hids from jobs manager - if filter_query_params.q and filter_query_params.qv and 'related-eq' in filter_query_params.q: - related_qv_index = filter_query_params.q.index('related-eq') - related_hid = filter_query_params.qv[related_qv_index] - job_connections_manager = JobConnectionsManager(trans.sa_session) - related = job_connections_manager.get_related_hids(trans, history_id, related_hid) - filter_query_params.qv[related_qv_index] = related - - filters = self.history_contents_filters.parse_query_filters(filter_query_params) + filters = self.history_contents_filters.parse_query_filters_with_relations(filter_query_params, history_id) stats_requested = accept == HistoryContentsWithStatsResult.__accept_type__ if stats_requested and self.history_contents_filters.contains_non_orm_filter(filters): diff --git a/test/unit/app/managers/test_JobConnectionsManager.py b/test/unit/app/managers/test_JobConnectionsManager.py index e63a9645519..2d400989a7f 100644 --- a/test/unit/app/managers/test_JobConnectionsManager.py +++ b/test/unit/app/managers/test_JobConnectionsManager.py @@ -1,7 +1,7 @@ import pytest from sqlalchemy import union -from galaxy.managers.jobs import JobConnectionsManager +from galaxy.managers.job_connections import JobConnectionsManager from galaxy.model import ( HistoryDatasetAssociation, HistoryDatasetCollectionAssociation, From cc83ec9d2f6c2e29557beb48b3d68faca5bf06f1 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Sun, 15 Jan 2023 19:15:14 +0500 Subject: [PATCH 10/15] Fix linting issues for job_connections --- lib/galaxy/managers/history_contents.py | 4 +- lib/galaxy/managers/job_connections.py | 46 ++++++++++++++----- .../managers/test_JobConnectionsManager.py | 22 +++++++-- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 3be17f5a0a3..22fcb71e232 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -529,8 +529,8 @@ class HistoryContentsFilters( def parse_query_filters_with_relations(self, query_filters: ValueFilterQueryParams, history_id): """Parse query filters but consider case where related filter is included.""" - if query_filters.q and query_filters.qv and 'related-eq' in query_filters.q: - qv_index = query_filters.q.index('related-eq') + if query_filters.q and query_filters.qv and "related-eq" in query_filters.q: + qv_index = query_filters.q.index("related-eq") qv_hid = query_filters.qv[qv_index] # Make new q and qv excluding related filter diff --git a/lib/galaxy/managers/job_connections.py b/lib/galaxy/managers/job_connections.py index 2a8254ab227..0392935c5ae 100644 --- a/lib/galaxy/managers/job_connections.py +++ b/lib/galaxy/managers/job_connections.py @@ -20,10 +20,10 @@ class JobConnectionsManager: def get_connections_graph(self, id, src): """Get connections graph of inputs and outputs for given item id""" - if src == 'HistoryDatasetAssociation': + if src == "HistoryDatasetAssociation": output_selects = self.outputs_derived_from_input_hda(id) input_selects = self.inputs_for_hda(id) - elif src == 'HistoryDatasetCollectionAssociation': + elif src == "HistoryDatasetCollectionAssociation": output_selects = self.outputs_derived_from_input_hdca(id) input_selects = self.inputs_for_hdca(id) else: @@ -41,15 +41,17 @@ class JobConnectionsManager: select(model.HistoryDatasetAssociation.id, expression.literal("HistoryDatasetAssociation")) .filter_by(history_id=history_id, hid=hid) .union( - select(model.HistoryDatasetCollectionAssociation.id, expression.literal("HistoryDatasetCollectionAssociation")) - .filter_by(history_id=history_id, hid=hid) + select( + model.HistoryDatasetCollectionAssociation.id, + expression.literal("HistoryDatasetCollectionAssociation"), + ).filter_by(history_id=history_id, hid=hid) ) ).all() result = [int(hid)] for item_data in items_by_hid: graph = self.get_connections_graph(id=item_data[0], src=item_data[1]) # Add found related items' hids to result list - for val in (graph["outputs"] + graph["inputs"]): + for val in graph["outputs"] + graph["inputs"]: item_class = get_class(val["src"]) item = self.sa_session.query(item_class).get(val["id"]) result.append(item.hid) @@ -63,7 +65,12 @@ class JobConnectionsManager: def outputs_derived_from_input_hda(self, input_hda_id): hda_select = ( - select([literal("HistoryDatasetAssociation").label("src"), model.JobToOutputDatasetAssociation.dataset_id.label("id")]) + select( + [ + literal("HistoryDatasetAssociation").label("src"), + model.JobToOutputDatasetAssociation.dataset_id.label("id"), + ] + ) .join( model.JobToInputDatasetAssociation, model.JobToInputDatasetAssociation.job_id == model.JobToOutputDatasetAssociation.job_id, @@ -87,7 +94,12 @@ class JobConnectionsManager: def outputs_derived_from_input_hdca(self, input_hdca_id): hda_select = ( - select([literal("HistoryDatasetAssociation").label("src"), model.JobToOutputDatasetAssociation.dataset_id.label("id")]) + select( + [ + literal("HistoryDatasetAssociation").label("src"), + model.JobToOutputDatasetAssociation.dataset_id.label("id"), + ] + ) .join( model.JobToInputDatasetCollectionAssociation, model.JobToInputDatasetCollectionAssociation.job_id == model.JobToOutputDatasetAssociation.job_id, @@ -103,7 +115,8 @@ class JobConnectionsManager: ) .join( model.JobToInputDatasetCollectionAssociation, - model.JobToInputDatasetCollectionAssociation.job_id == model.JobToOutputDatasetCollectionAssociation.job_id, + model.JobToInputDatasetCollectionAssociation.job_id + == model.JobToOutputDatasetCollectionAssociation.job_id, ) .where(model.JobToInputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) ) @@ -111,7 +124,12 @@ class JobConnectionsManager: def inputs_for_hda(self, input_hda_id): input_hdas = ( - select([literal("HistoryDatasetAssociation").label("src"), model.JobToInputDatasetAssociation.dataset_id.label("id")]) + select( + [ + literal("HistoryDatasetAssociation").label("src"), + model.JobToInputDatasetAssociation.dataset_id.label("id"), + ] + ) .join( model.JobToOutputDatasetAssociation, model.JobToOutputDatasetAssociation.job_id == model.JobToInputDatasetAssociation.job_id, @@ -135,7 +153,12 @@ class JobConnectionsManager: def inputs_for_hdca(self, input_hdca_id): input_hdas = ( - select([literal("HistoryDatasetAssociation").label("src"), model.JobToInputDatasetAssociation.dataset_id.label("id")]) + select( + [ + literal("HistoryDatasetAssociation").label("src"), + model.JobToInputDatasetAssociation.dataset_id.label("id"), + ] + ) .join( model.JobToOutputDatasetCollectionAssociation, model.JobToOutputDatasetCollectionAssociation.job_id == model.JobToInputDatasetAssociation.job_id, @@ -151,7 +174,8 @@ class JobConnectionsManager: ) .join( model.JobToOutputDatasetCollectionAssociation, - model.JobToOutputDatasetCollectionAssociation.job_id == model.JobToInputDatasetCollectionAssociation.job_id, + model.JobToOutputDatasetCollectionAssociation.job_id + == model.JobToInputDatasetCollectionAssociation.job_id, ) .where(model.JobToOutputDatasetCollectionAssociation.dataset_collection_id == input_hdca_id) ) diff --git a/test/unit/app/managers/test_JobConnectionsManager.py b/test/unit/app/managers/test_JobConnectionsManager.py index 2d400989a7f..844b4dbdcdf 100644 --- a/test/unit/app/managers/test_JobConnectionsManager.py +++ b/test/unit/app/managers/test_JobConnectionsManager.py @@ -40,8 +40,14 @@ def setup_connected_dataset(sa_session: galaxy_scoped_session): sa_session.add_all([center_hda, input_hda, input_hdca, output_hdca, input_job, output_job]) sa_session.flush() expected_graph = { - "inputs": [{"src": "HistoryDatasetAssociation", "id": input_hda.id}, {"src": "HistoryDatasetCollectionAssociation", "id": input_hdca.id}], - "outputs": [{"src": "HistoryDatasetAssociation", "id": output_hda.id}, {"src": "HistoryDatasetCollectionAssociation", "id": output_hdca.id}], + "inputs": [ + {"src": "HistoryDatasetAssociation", "id": input_hda.id}, + {"src": "HistoryDatasetCollectionAssociation", "id": input_hdca.id}, + ], + "outputs": [ + {"src": "HistoryDatasetAssociation", "id": output_hda.id}, + {"src": "HistoryDatasetCollectionAssociation", "id": output_hdca.id}, + ], } return center_hda, expected_graph @@ -68,9 +74,12 @@ def setup_connected_dataset_collection(sa_session: galaxy_scoped_session): "inputs": [ {"src": "HistoryDatasetAssociation", "id": input_hda1.id}, {"src": "HistoryDatasetAssociation", "id": input_hda2.id}, - {"src": "HistoryDatasetCollectionAssociation", "id": input_hdca.id} + {"src": "HistoryDatasetCollectionAssociation", "id": input_hdca.id}, + ], + "outputs": [ + {"src": "HistoryDatasetAssociation", "id": output_hda.id}, + {"src": "HistoryDatasetCollectionAssociation", "id": output_hdca.id}, ], - "outputs": [{"src": "HistoryDatasetAssociation", "id": output_hda.id}, {"src": "HistoryDatasetCollectionAssociation", "id": output_hdca.id}], } return center_hdca, expected_graph @@ -111,4 +120,7 @@ def test_graph_manager_hda(job_connections_manager: JobConnectionsManager): def test_graph_manager_hdca(job_connections_manager: JobConnectionsManager): center_hdca, expected_graph = setup_connected_dataset_collection(job_connections_manager.sa_session) - assert job_connections_manager.get_connections_graph(center_hdca.id, "HistoryDatasetCollectionAssociation") == expected_graph + assert ( + job_connections_manager.get_connections_graph(center_hdca.id, "HistoryDatasetCollectionAssociation") + == expected_graph + ) From a151791d78e2ec36044c46c84d75e8e30c29aed9 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Sun, 15 Jan 2023 19:22:56 +0500 Subject: [PATCH 11/15] Change conversion to list for related filter --- lib/galaxy/managers/history_contents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 22fcb71e232..3e988f1dccf 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -566,7 +566,7 @@ class HistoryContentsFilters( if attr == "related" and op == "eq": if isinstance(val, str): - val = val.strip('][').split(', ') + val = val.strip("][").split(", ") return sql.column("hid").in_(val) if attr == "type_id": From 0d97f62cdc3875ca941738412ba89cc3368a26a9 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Sun, 15 Jan 2023 20:33:44 +0500 Subject: [PATCH 12/15] Fix import ordering for job_connections managers --- lib/galaxy/managers/history_contents.py | 2 +- lib/galaxy/managers/job_connections.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 3e988f1dccf..f00be5bcc87 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -32,7 +32,6 @@ from galaxy import ( exceptions as glx_exceptions, model, ) -from galaxy.schema import ValueFilterQueryParams from galaxy.managers import ( annotatable, base, @@ -43,6 +42,7 @@ from galaxy.managers import ( tools, ) from galaxy.managers.job_connections import JobConnectionsManager +from galaxy.schema import ValueFilterQueryParams from galaxy.structured_app import MinimalManagerApp from .base import ( parse_bool, diff --git a/lib/galaxy/managers/job_connections.py b/lib/galaxy/managers/job_connections.py index 0392935c5ae..8710984aaa6 100644 --- a/lib/galaxy/managers/job_connections.py +++ b/lib/galaxy/managers/job_connections.py @@ -3,8 +3,8 @@ from sqlalchemy import ( union, ) from sqlalchemy.sql import ( - select, expression, + select, ) from galaxy import model From ecb518d43232271dedd8736539790b8380849fd6 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Sun, 15 Jan 2023 23:25:33 +0500 Subject: [PATCH 13/15] Improve type conversion for related job_connections filter --- lib/galaxy/managers/history_contents.py | 7 +++---- lib/galaxy/managers/job_connections.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index f00be5bcc87..055a8b94687 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -2,6 +2,7 @@ Heterogenous lists/contents are difficult to query properly since unions are not easily made. """ +import json import logging from typing import ( Any, @@ -543,7 +544,7 @@ class HistoryContentsFilters( # Make new query_filters with updated list of related hids for given hid new_q.append("related-eq") - new_qv.append(str(related)) + new_qv.append(json.dumps(related)) query_filters_with_relations = ValueFilterQueryParams( q=new_q, qv=new_qv, @@ -565,9 +566,7 @@ class HistoryContentsFilters( raise_filter_err(attr, op, val, "bad op in filter") if attr == "related" and op == "eq": - if isinstance(val, str): - val = val.strip("][").split(", ") - return sql.column("hid").in_(val) + return sql.column("hid").in_(json.loads(val)) if attr == "type_id": if op == "eq": diff --git a/lib/galaxy/managers/job_connections.py b/lib/galaxy/managers/job_connections.py index 8710984aaa6..c52480c93d2 100644 --- a/lib/galaxy/managers/job_connections.py +++ b/lib/galaxy/managers/job_connections.py @@ -47,7 +47,7 @@ class JobConnectionsManager: ).filter_by(history_id=history_id, hid=hid) ) ).all() - result = [int(hid)] + result = [hid] for item_data in items_by_hid: graph = self.get_connections_graph(id=item_data[0], src=item_data[1]) # Add found related items' hids to result list From 49cf3eef9e5883fe44dc845929f3f4bcc83e6a22 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Mon, 16 Jan 2023 17:51:56 +0500 Subject: [PATCH 14/15] Remove previous highlights.js (history i/o) file --- .../History/Content/model/highlights.js | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 client/src/components/History/Content/model/highlights.js diff --git a/client/src/components/History/Content/model/highlights.js b/client/src/components/History/Content/model/highlights.js deleted file mode 100644 index 7ae21b5905a..00000000000 --- a/client/src/components/History/Content/model/highlights.js +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Specifies highlighted items in the history listing. The `highlight` property is passed to - * the content item component and can be used to modify its appearance. - * TO DO: Consider case where parameter history is different and hence inputs cannot be seen - * in the current panel. - */ -import axios from "axios"; -import { prependPath } from "utils/redirect"; -import { deepeach } from "utils/utils"; -import { LastQueue } from "utils/promise-queue"; - -// add promise queue -const lastQueue = new LastQueue(300); - -/** Local cache for parameter requests */ -const paramStash = new Map(); - -/** Performs request to obtain dataset parameters */ -async function getDatasetParameters(datasetId, jobId) { - if (!paramStash.has(datasetId)) { - const url = jobId - ? `api/jobs/${jobId}/parameters_display` - : `api/datasets/${datasetId}/parameters_display?hda_ldda=hda`; - const { data } = await lastQueue.enqueue(axios.get, prependPath(url)); - paramStash.set(datasetId, data); - } - return paramStash.get(datasetId); -} - -/** Returns item key */ -function getKey(details) { - if (details.id && details.src) { - const historyContentType = details.src == "hda" ? "dataset" : "dataset_collection"; - return `${details.id}-${historyContentType}`; - } - return null; -} - -/** Returns highlighting details */ -export async function getHighlights(item, itemKey) { - const highlights = {}; - const { outputs, parameters } = await getDatasetParameters(item.id, item.job_source_id); - deepeach(parameters, (details) => { - const key = getKey(details); - if (key) { - highlights[key] = "input"; - } - }); - deepeach(outputs, (details) => { - const key = getKey(details); - if (key) { - // some other item created this item (e.g.: inheritance) - if (key != itemKey) { - highlights[itemKey] = "output"; - highlights[key] = "input"; - } else { - highlights[key] = "output"; - } - } - }); - // highlights only has item itself as an output (i.e.: no inputs) - if (highlights[itemKey] === "output" && Object.keys(highlights).length == 1) { - highlights[itemKey] = "noInputs"; - } - // TO DO: Consider case where a job created multiple items (all highlights are outputs) - return highlights; -} From 06a43ad282cb474d23604a05b170dbcdd5cba070 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Mon, 16 Jan 2023 19:54:31 +0500 Subject: [PATCH 15/15] Remove unused getItemKey function --- client/src/components/History/CurrentHistory/HistoryPanel.vue | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/src/components/History/CurrentHistory/HistoryPanel.vue b/client/src/components/History/CurrentHistory/HistoryPanel.vue index 9c176101c9c..cb453905505 100644 --- a/client/src/components/History/CurrentHistory/HistoryPanel.vue +++ b/client/src/components/History/CurrentHistory/HistoryPanel.vue @@ -291,9 +291,6 @@ export default { return null; } }, - getItemKey(item) { - return `${item.id}-${item.history_content_type}`; - }, hasMatches(items) { return !!items && items.length > 0; },