Merge pull request #13867 from jmchilton/workflow_index_fixes

Workflow Index Fixes and Improvements
This commit is contained in:
Björn Grüning
2022-05-06 00:06:36 +02:00
committed by GitHub
9 changed files with 377 additions and 147 deletions
@@ -62,8 +62,18 @@
<Tags :index="row.index" :tags="row.item.tags" @input="onTags" @tag-click="onTagClick" />
</template>
<template v-slot:cell(published)="row">
<font-awesome-icon v-if="row.item.published" v-b-tooltip.hover title="Published" icon="globe" />
<font-awesome-icon v-if="row.item.shared" v-b-tooltip.hover title="Shared" icon="share-alt" />
<font-awesome-icon
v-if="row.item.published"
v-b-tooltip.hover
title="Published"
icon="globe"
@click="appendFilter('is:published')" />
<font-awesome-icon
v-if="row.item.shared"
v-b-tooltip.hover
title="Shared"
icon="share-alt"
@click="appendFilter('is:shared_with_me')" />
</template>
<template v-slot:cell(show_in_tool_panel)="row">
<b-link @click="bookmarkWorkflow(row.item, false)" v-if="row.item.show_in_tool_panel">
@@ -246,11 +256,14 @@ export default {
},
onTagClick: function (tag) {
const tagFilter = `tag:'${tag.text}'`;
this.appendFilter(tagFilter);
},
appendFilter(text) {
const initialFilter = this.filter;
if (initialFilter.length === 0) {
this.filter = tagFilter;
} else if (initialFilter.indexOf(tagFilter) < 0) {
this.filter = `${tagFilter} ${initialFilter}`;
this.filter = text;
} else if (initialFilter.indexOf(text) < 0) {
this.filter = `${text} ${initialFilter}`;
}
},
onAdd: function (workflow) {
@@ -124,5 +124,31 @@ describe("WorkflowList.vue", () => {
flushPromises();
expect(wrapper.vm.filter).toBe("tag:'tagmoo'");
});
it("update filter when published icon is clicked", async () => {
axiosMock
.onGet("/api/workflows", {
params: { limit: 50, offset: 0, skip_step_counts: true, search: "is:published" },
})
.reply(200, mockWorkflowsData, { total_matches: "1" });
const rows = wrapper.findAll("tbody > tr").wrappers;
const row = rows[0];
row.find(".fa-globe").trigger("click");
flushPromises();
expect(wrapper.vm.filter).toBe("is:published");
});
it("update filter when shared with me icon is clicked", async () => {
axiosMock
.onGet("/api/workflows", {
params: { limit: 50, offset: 0, skip_step_counts: true, search: "is:shared_with_me" },
})
.reply(200, mockWorkflowsData, { total_matches: "1" });
const rows = wrapper.findAll("tbody > tr").wrappers;
const row = rows[0];
row.find(".fa-share-alt").trigger("click");
flushPromises();
expect(wrapper.vm.filter).toBe("is:shared_with_me");
});
});
});
+136 -3
View File
@@ -9,6 +9,7 @@ from typing import (
Dict,
List,
Optional,
Tuple,
)
from gxformat2 import (
@@ -18,9 +19,17 @@ from gxformat2 import (
python_to_workflow,
)
from pydantic import BaseModel
from sqlalchemy import and_
from sqlalchemy import (
and_,
desc,
false,
or_,
true,
)
from sqlalchemy.orm import (
aliased,
joinedload,
Query,
subqueryload,
)
@@ -31,7 +40,14 @@ from galaxy import (
)
from galaxy.job_execution.actions.post import ActionBox
from galaxy.managers import sharable
from galaxy.managers.context import ProvidesUserContext
from galaxy.model.index_filter_util import (
raw_text_column_filter,
tag_filter,
text_column_filter,
)
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.schema.schema import WorkflowIndexQueryPayload
from galaxy.structured_app import MinimalManagerApp
from galaxy.tools.parameters import (
params_to_incoming,
@@ -48,6 +64,11 @@ from galaxy.util.json import (
safe_loads,
)
from galaxy.util.sanitize_html import sanitize_html
from galaxy.util.search import (
FilteredTerm,
parse_filters_structured,
RawTextTerm,
)
from galaxy.web import url_for
from galaxy.workflow.modules import (
is_tool_module_type,
@@ -69,6 +90,15 @@ from .executables import artifact_class
log = logging.getLogger(__name__)
INDEX_SEARCH_FILTERS = {
"name": "name",
"tag": "tag",
"n": "name",
"t": "tag",
"is": "is",
}
class WorkflowsManager(sharable.SharableModelManager):
"""Handle CRUD type operations related to workflows. More interesting
stuff regarding workflow execution, step sorting, etc... can be found in
@@ -83,6 +113,109 @@ class WorkflowsManager(sharable.SharableModelManager):
super().__init__(app)
self.app = app
def index_query(
self, trans: ProvidesUserContext, payload: WorkflowIndexQueryPayload, include_total_count: bool = False
) -> Tuple[Query, Optional[int]]:
show_published = payload.show_published
show_hidden = payload.show_hidden
show_deleted = payload.show_deleted
show_shared = payload.show_shared
if show_shared is None:
show_shared = not show_hidden and not show_deleted
if show_shared and show_deleted:
message = "show_shared and show_deleted cannot both be specified as true"
raise exceptions.RequestParameterInvalidException(message)
if show_shared and show_hidden:
message = "show_shared and show_hidden cannot both be specified as true"
raise exceptions.RequestParameterInvalidException(message)
filters = [
model.StoredWorkflow.user == trans.user,
]
user = trans.user
if user and show_shared:
filters.append(model.StoredWorkflowUserShareAssociation.user == user)
if show_published or user is None and show_published is None:
filters.append((model.StoredWorkflow.published == true()))
query = trans.sa_session.query(model.StoredWorkflow)
if show_shared:
query = query.outerjoin(model.StoredWorkflow.users_shared_with)
query = query.outerjoin(model.StoredWorkflow.tags)
latest_workflow_load = joinedload("latest_workflow")
if not payload.skip_step_counts:
latest_workflow_load = latest_workflow_load.undefer("step_count")
latest_workflow_load = latest_workflow_load.lazyload("steps")
query = query.options(joinedload(model.StoredWorkflow.annotations))
query = query.options(latest_workflow_load)
query = query.filter(or_(*filters))
query = query.filter(model.StoredWorkflow.table.c.hidden == (true() if show_hidden else false()))
query = query.filter(model.StoredWorkflow.table.c.deleted == (true() if show_deleted else false()))
if payload.search:
search_query = payload.search
parsed_search = parse_filters_structured(search_query, INDEX_SEARCH_FILTERS)
def w_tag_filter(term_text: str, quoted: bool):
nonlocal query
alias = aliased(model.StoredWorkflowTagAssociation)
query = query.outerjoin(model.StoredWorkflow.tags.of_type(alias))
return tag_filter(alias, term_text, quoted)
def name_filter(term):
return text_column_filter(model.StoredWorkflow.name, term)
for term in parsed_search.terms:
if isinstance(term, FilteredTerm):
key = term.filter
q = term.text
if key == "tag":
tf = w_tag_filter(term.text, term.quoted)
query = query.filter(tf)
elif key == "name":
query = query.filter(name_filter(term))
elif key == "is":
if q == "published":
query = query.filter(model.StoredWorkflow.published == true())
elif q == "shared_with_me":
if not show_shared:
message = "Can only use tag is:shared_with_me if show_shared parameter also true."
raise exceptions.RequestParameterInvalidException(message)
query = query.filter(model.StoredWorkflowUserShareAssociation.user == user)
elif isinstance(term, RawTextTerm):
tf = w_tag_filter(term.text, False)
query = query.filter(
raw_text_column_filter(
[
model.StoredWorkflow.name,
tf,
],
term,
)
)
if include_total_count:
total_matches = query.count()
else:
total_matches = None
if payload.sort_by is None:
if user:
query = query.order_by(desc(model.StoredWorkflow.user == user))
query = query.order_by(desc(model.StoredWorkflow.table.c.update_time))
else:
sort_column = getattr(model.StoredWorkflow, payload.sort_by)
if payload.sort_desc:
sort_column = sort_column.desc()
query = query.order_by(sort_column)
if payload.limit is not None:
query = query.limit(payload.limit)
if payload.offset is not None:
query = query.offset(payload.offset)
return query, total_matches
def get_stored_workflow(self, trans, workflow_id, by_stored_id=True):
"""Use a supplied ID (UUID or encoded stored workflow ID) to find
a workflow.
@@ -286,7 +419,7 @@ class WorkflowsManager(sharable.SharableModelManager):
def build_invocations_query(
self,
trans,
trans: ProvidesUserContext,
stored_workflow_id=None,
history_id=None,
job_id=None,
@@ -296,7 +429,7 @@ class WorkflowsManager(sharable.SharableModelManager):
offset=None,
sort_by=None,
sort_desc=None,
):
) -> Tuple[Query, int]:
"""Get invocations owned by the current user."""
sa_session = trans.sa_session
invocations_query = sa_session.query(model.WorkflowInvocation)
+53
View File
@@ -0,0 +1,53 @@
"""Utility functions used to adapt galaxy.util.search to Galaxy model index queries."""
from typing import (
List,
Union,
)
from sqlalchemy import (
and_,
or_,
)
from sqlalchemy.orm import InstrumentedAttribute
from sqlalchemy.sql.expression import BinaryExpression
from galaxy.util.search import (
FilteredTerm,
RawTextTerm,
)
def text_column_filter(column, term: FilteredTerm):
if term.quoted:
filter = column == term.text
else:
filter = column.ilike(f"%{term.text}%")
return filter
RawTextSearchableT = Union[BinaryExpression, InstrumentedAttribute]
def raw_text_column_filter(columns: List[RawTextSearchableT], term: RawTextTerm):
like_text = f"%{term.text}%"
return or_(*[c.ilike(like_text) if isinstance(c, InstrumentedAttribute) else c for c in columns])
def tag_filter(assocation_model_class, term_text, quoted: bool = False):
if ":" in term_text:
key, value = term_text.rsplit(":", 1)
if not quoted:
return and_(
assocation_model_class.user_tname.ilike(key),
assocation_model_class.user_value.ilike(f"%{value}%"),
)
else:
return and_(
assocation_model_class.user_tname == key,
assocation_model_class.user_value == value,
)
else:
if not quoted:
return assocation_model_class.user_tname.ilike(f"%{term_text}%")
else:
return assocation_model_class.user_tname == term_text
+1 -2
View File
@@ -1048,12 +1048,11 @@ class WorkflowSortByEnum(str, Enum):
none = None
class WorkflowIndexPayload(Model):
class WorkflowIndexQueryPayload(Model):
show_deleted: bool = False
show_hidden: bool = False
show_published: Optional[bool] = None
show_shared: Optional[bool] = None
missing_tools: bool = False
sort_by: Optional[WorkflowSortByEnum] = Field(title="Sort By", description="Sort workflows by this attribute")
sort_desc: Optional[bool] = Field(
title="Sort descending", description="Explicitly sort by descending if sort_by is specified."
+57
View File
@@ -3,10 +3,12 @@ This module *does not* contain API routes. It exclusively contains dependencies
"""
import inspect
from enum import Enum
from string import Template
from typing import (
Any,
AsyncGenerator,
cast,
NamedTuple,
Optional,
Tuple,
Type,
@@ -410,3 +412,58 @@ async def try_get_request_body_as_json(request: Request) -> Optional[Any]:
body = await request.json()
return body
return None
search_description_template = Template(
"""A mix of free text and GitHub-style tags used to filter the index operation.
## Query Structure
GitHub-style filter tags (not be confused with Galaxy tags) are tags of the form
`<tag_name>:<text_no_spaces>` or `<tag_name>:'<text with potential spaces>'`. The tag name
*generally* (but not exclusively) corresponds to the name of an attribute on the model
being indexed (i.e. a column in the database).
If the tag is quoted, the attribute will be filtered exactly. If the tag is unquoted,
generally a partial match will be used to filter the query (i.e. in terms of the implementation
this means the database operation `ILIKE` will typically be used).
Once the tagged filters are extracted from the search query, the remaing text is just
used to search various documented attributes of the object.
## GitHub-style Tags Available
${tags}
## Free Text
Free text search terms will be searched against the following attributes of the
${model_name}s: ${freetext}.
"""
)
class IndexQueryTag(NamedTuple):
tag: str
description: str
alias: Optional[str] = None
def as_markdown(self):
desc = self.description
alias = self.alias
if alias:
desc += f" (The tag `{alias}` can be used a short hand alias for this tag to filter on this attribute.)"
return f"`{self.tag}`\n: {desc}"
def search_query_param(model_name: str, tags: list, free_text_fields: list) -> Optional[str]:
tags_markdown_str = "\n\n".join([t.as_markdown() for t in tags])
description = search_description_template.safe_substitute(
model_name=model_name, tags=tags_markdown_str, freetext=", ".join([f"`{t}`" for t in free_text_fields])
)
return Query(
default=None,
title="Search query.",
description=description,
)
+28 -7
View File
@@ -48,7 +48,6 @@ from galaxy.schema.schema import (
ShareWithPayload,
ShareWithStatus,
SharingStatus,
WorkflowIndexPayload,
WorkflowSortByEnum,
)
from galaxy.structured_app import StructuredApp
@@ -76,7 +75,10 @@ from galaxy.webapps.galaxy.services.invocations import (
InvocationSerializationParams,
InvocationsService,
)
from galaxy.webapps.galaxy.services.workflows import WorkflowsService
from galaxy.webapps.galaxy.services.workflows import (
WorkflowIndexPayload,
WorkflowsService,
)
from galaxy.workflow.extract import extract_workflow
from galaxy.workflow.modules import module_factory
from galaxy.workflow.run import queue_invoke
@@ -85,7 +87,9 @@ from . import (
BaseGalaxyAPIController,
depends,
DependsOnTrans,
IndexQueryTag,
Router,
search_query_param,
)
log = logging.getLogger(__name__)
@@ -1444,10 +1448,27 @@ OffsetQueryParam: Optional[int] = Query(
title="Number of workflows to skip in sorted query (to enable pagination).",
)
SearchQueryParam: Optional[str] = Query(
default=None,
title="Search query.",
description="Free text used to filter the query. Filters on a name and tags, Github-style tags can be used to be more specific. Free-text search is case-insensitive.",
query_tags = [
IndexQueryTag("name", "The stored workflow's name.", "n"),
IndexQueryTag(
"tag",
"The workflow's tag, if the tag contains a colon an approach will be made to match the key and value of the tag separately.",
"t",
),
IndexQueryTag(
"is:published",
"Include only published workflows in the final result. Be sure the the query parameter `show_published` is set to `true` if to include all published workflows and not just the requesting user's.",
),
IndexQueryTag(
"is:share_with_me",
"Include only workflows shared with the requesting user. Be sure the the query parameter `show_shared` is set to `true` if to include shared workflows.",
),
]
SearchQueryParam: Optional[str] = search_query_param(
model_name="Stored Workflow",
tags=query_tags,
free_text_fields=["name", "tag"],
)
SkipStepCountsQueryParam: bool = Query(
@@ -1482,7 +1503,7 @@ class FastAPIWorkflows:
search: Optional[str] = SearchQueryParam,
skip_step_counts: bool = SkipStepCountsQueryParam,
) -> List[Dict[str, Any]]:
"""Return the sharing status of the item."""
"""Lists stored workflows viewable by the user."""
payload = WorkflowIndexPayload(
show_published=show_published,
show_hidden=show_hidden,
+7 -130
View File
@@ -7,45 +7,23 @@ from typing import (
Tuple,
)
from sqlalchemy import (
and_,
desc,
false,
or_,
true,
)
from sqlalchemy.orm import joinedload
from galaxy import (
exceptions,
model,
web,
)
from galaxy import web
from galaxy.managers.context import ProvidesUserContext
from galaxy.managers.workflows import (
WorkflowContentsManager,
WorkflowSerializer,
WorkflowsManager,
)
from galaxy.schema.schema import WorkflowIndexPayload
from galaxy.schema.schema import WorkflowIndexQueryPayload
from galaxy.tool_shed.tool_shed_registry import Registry
from galaxy.util.search import (
FilteredTerm,
parse_filters_structured,
RawTextTerm,
)
from galaxy.webapps.galaxy.services.base import ServiceBase
from galaxy.webapps.galaxy.services.sharable import ShareableService
log = logging.getLogger(__name__)
INDEX_SEARCH_FILTERS = {
"name": "name",
"tag": "tag",
"n": "name",
"t": "tag",
}
class WorkflowIndexPayload(WorkflowIndexQueryPayload):
missing_tools: bool = False
class WorkflowsService(ServiceBase):
@@ -68,111 +46,10 @@ class WorkflowsService(ServiceBase):
payload: WorkflowIndexPayload,
include_total_count: bool = False,
) -> Tuple[List[Dict[str, Any]], Optional[int]]:
show_published = payload.show_published
show_hidden = payload.show_hidden
show_deleted = payload.show_deleted
missing_tools = payload.missing_tools
show_shared = payload.show_shared
if show_shared is None:
show_shared = not show_hidden and not show_deleted
if show_shared and show_deleted:
message = "show_shared and show_deleted cannot both be specified as true"
raise exceptions.RequestParameterInvalidException(message)
if show_shared and show_hidden:
message = "show_shared and show_hidden cannot both be specified as true"
raise exceptions.RequestParameterInvalidException(message)
rval = []
filters = [
model.StoredWorkflow.user == trans.user,
]
user = trans.user
if user and show_shared:
filters.append(model.StoredWorkflowUserShareAssociation.user == user)
if show_published or user is None and show_published is None:
filters.append((model.StoredWorkflow.published == true()))
query = trans.sa_session.query(model.StoredWorkflow)
if show_shared:
query = query.outerjoin(model.StoredWorkflow.users_shared_with)
query = query.outerjoin(model.StoredWorkflow.tags)
latest_workflow_load = joinedload("latest_workflow")
if not payload.skip_step_counts:
latest_workflow_load = latest_workflow_load.undefer("step_count")
latest_workflow_load = latest_workflow_load.lazyload("steps")
query = query.options(joinedload(model.StoredWorkflow.annotations))
query = query.options(latest_workflow_load)
query = query.filter(or_(*filters))
query = query.filter(model.StoredWorkflow.table.c.hidden == (true() if show_hidden else false()))
query = query.filter(model.StoredWorkflow.table.c.deleted == (true() if show_deleted else false()))
if payload.search:
search_query = payload.search
parsed_search = parse_filters_structured(search_query, INDEX_SEARCH_FILTERS)
def tag_filter(term_text: str, quoted: bool):
term_text = term.text
if ":" in term_text:
key, value = term_text.rsplit(":", 1)
if not quoted:
return and_(
model.StoredWorkflowTagAssociation.user_tname.ilike(key),
model.StoredWorkflowTagAssociation.user_value.ilike(f"%{value}%"),
)
else:
return and_(
model.StoredWorkflowTagAssociation.user_tname == key,
model.StoredWorkflowTagAssociation.user_value == value,
)
else:
if not quoted:
return model.StoredWorkflowTagAssociation.user_tname.ilike(f"%{term_text}%")
else:
return model.StoredWorkflowTagAssociation.user_tname == term_text
def name_filter(text, quoted):
if not quoted:
filter = model.StoredWorkflow.name.ilike(f"%{text}%")
else:
filter = model.StoredWorkflow.name == text
return filter
for term in parsed_search.terms:
if isinstance(term, FilteredTerm):
key = term.filter
q = term.text
if key == "tag":
query = query.filter(tag_filter(term.text, term.quoted))
elif key == "name":
query = query.filter(name_filter(q, term.quoted))
elif isinstance(term, RawTextTerm):
query = query.filter(
or_(
name_filter(term.text, False),
tag_filter(term.text, False),
)
)
if include_total_count:
total_matches = query.count()
else:
total_matches = None
if payload.sort_by is None:
if user:
query = query.order_by(desc(model.StoredWorkflow.user == user))
query = query.order_by(desc(model.StoredWorkflow.table.c.update_time))
else:
sort_column = getattr(model.StoredWorkflow, payload.sort_by)
if payload.sort_desc:
sort_column = sort_column.desc()
query = query.order_by(sort_column)
if payload.limit is not None:
query = query.limit(payload.limit)
if payload.offset is not None:
query = query.offset(payload.offset)
missing_tools = payload.missing_tools
query, total_matches = self._workflows_manager.index_query(trans, payload, include_total_count)
rval = []
for wf in query.all():
item = wf.to_dict(value_mapper={"id": trans.security.encode_id})
encoded_id = trans.security.encode_id(wf.id)
+51
View File
@@ -493,6 +493,39 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase):
index_ids = self.workflow_populator.index_ids(search="tag:moocowatag")
assert workflow_id_1 in index_ids
def test_index_search_tags_multiple(self):
name1 = self.dataset_populator.get_random_name()
name2 = self.dataset_populator.get_random_name()
name3 = self.dataset_populator.get_random_name()
workflow_id_1 = self.workflow_populator.simple_workflow(name1)
workflow_id_2 = self.workflow_populator.simple_workflow(name2)
workflow_id_3 = self.workflow_populator.simple_workflow(name3)
self.workflow_populator.set_tags(workflow_id_1, ["multipletagfilter1", "multipletagfilter2", "decoy1"])
self.workflow_populator.set_tags(workflow_id_2, ["multipletagfilter1", "decoy2"])
self.workflow_populator.set_tags(workflow_id_3, ["multipletagfilter2", "decoy3"])
for search in ["multipletagfilter1", "tag:ipletagfilter1", "tag:'multipletagfilter1'"]:
index_ids = self.workflow_populator.index_ids(search=search)
assert workflow_id_1 in index_ids
assert workflow_id_2 in index_ids
assert workflow_id_3 not in index_ids
for search in ["multipletagfilter2", "tag:ipletagfilter2", "tag:'multipletagfilter2'"]:
index_ids = self.workflow_populator.index_ids(search=search)
assert workflow_id_1 in index_ids
assert workflow_id_2 not in index_ids
assert workflow_id_3 in index_ids
for search in [
"multipletagfilter2 multipletagfilter1",
"tag:filter2 tag:tagfilter1",
"tag:'multipletagfilter2' tag:'multipletagfilter1'",
]:
index_ids = self.workflow_populator.index_ids(search=search)
assert workflow_id_1 in index_ids
assert workflow_id_2 not in index_ids
assert workflow_id_3 not in index_ids
def test_search_casing(self):
name1, name2 = (
self.dataset_populator.get_random_name().upper(),
@@ -538,6 +571,24 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase):
assert workflow_id in self.workflow_populator.index_ids(show_published=True)
assert workflow_id not in self.workflow_populator.index_ids(show_published=False)
def test_index_search_is_tags(self):
my_workflow_id_1 = self.workflow_populator.simple_workflow("sitags_m_1")
my_email = self.dataset_populator.user_email()
with self._different_user():
their_workflow_id_1 = self.workflow_populator.simple_workflow("sitags_shwm_1")
self.workflow_populator.share_with_user(their_workflow_id_1, my_email)
published_workflow_id_1 = self.workflow_populator.simple_workflow("sitags_p_1", publish=True)
index_ids = self.workflow_populator.index_ids(search="is:published", show_published=True)
assert published_workflow_id_1 in index_ids
assert their_workflow_id_1 not in index_ids
assert my_workflow_id_1 not in index_ids
index_ids = self.workflow_populator.index_ids(search="is:shared_with_me")
assert published_workflow_id_1 not in index_ids
assert their_workflow_id_1 in index_ids
assert my_workflow_id_1 not in index_ids
def test_index_parameter_invalid_combinations(self):
# these can all be called by themselves and return 200...
response = self._get("workflows?show_hidden=true")