From 57cd9f369e8068bd9ba46cf9a03e7b7f74c63d78 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 21 Jun 2019 12:53:10 +0200 Subject: [PATCH 1/7] filters: add regexp filter and efficient sort - also started to add docs - use likely more efficient implementation of the sort filter. used insertion sort before which is n^2 (n^3 if the insertions take linear time) --- lib/galaxy/tool_util/xsd/galaxy.xsd | 20 +++++-- .../tools/parameters/dynamic_options.py | 53 +++++++++++++++---- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index cea5aa9e6f6..9bbc11933b7 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -3599,12 +3599,24 @@ demonstrates splitting up strings into multiple values. +in the ``filter_types`` dictionary + + ]]> diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index 6f6acb5b8b8..92ff35ac036 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -4,6 +4,7 @@ on the values of other parameters or other aspects of the current state) """ import logging import os +import re from six import StringIO @@ -74,7 +75,44 @@ class StaticValueFilter(Filter): except Exception: pass for fields in options: - if (self.keep and fields[self.column] == filter_value) or (not self.keep and fields[self.column] != filter_value): + if self.keep == (filter_value == fields[self.column]): + rval.append(fields) + return rval + + +class RegexpFilter(Filter): + """ + Filters a list of options on a column by a regular expression. + + Type: regexp + + Required Attributes: + value: regular expression to compare to + column: column in options to compare with + Optional Attributes: + keep: Keep columns matching the regexp (True) + Discard columns matching the regexp (False) + """ + + def __init__(self, d_option, elem): + Filter.__init__(self, d_option, elem) + self.value = elem.get("value", None) + assert self.value is not None, "Required 'value' attribute missing from filter" + column = elem.get("column", None) + assert column is not None, "Required 'column' attribute missing from filter, when loading from file" + self.column = d_option.column_spec_to_index(column) + self.keep = string_as_bool(elem.get("keep", 'True')) + + def filter_options(self, options, trans, other_values): + rval = [] + filter_value = self.value + try: + filter_value = User.expand_user_properties(trans.user, filter_value) + except Exception: + pass + filter_pattern = re.compile(filter_value) + for fields in options: + if self.keep == (not filter_pattern.match(fields[self.column]) is None): rval.append(fields) return rval @@ -226,7 +264,7 @@ class ParamValueFilter(Filter): ref = str(ref) rval = [] for fields in options: - if (self.keep and fields[self.column] == ref) or (not self.keep and fields[self.column] != ref): + if self.keep == (fields[self.column] == ref): rval.append(fields) return rval @@ -442,20 +480,13 @@ class SortByColumnFilter(Filter): self.column = d_option.column_spec_to_index(column) def filter_options(self, options, trans, other_values): - rval = [] - for fields in options: - for j in range(0, len(rval)): - if fields[self.column] < rval[j][self.column]: - rval.insert(j, fields) - break - else: - rval.append(fields) - return rval + return sorted(options, key=lambda x: x[self.column]) filter_types = dict(data_meta=DataMetaFilter, param_value=ParamValueFilter, static_value=StaticValueFilter, + regexp=RegexpFilter, unique_value=UniqueValueFilter, multiple_splitter=MultipleSplitterFilter, attribute_value_splitter=AttributeValueSplitterFilter, From 64deb4dbc08b24da2e4a1f071e00a4c8b7dbf3b4 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 16 Aug 2019 17:15:44 +0200 Subject: [PATCH 2/7] extend test for dbkey filtering code suggest that column names can be used, lets try --- test/functional/tools/dbkey_filter_multi_input.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/tools/dbkey_filter_multi_input.xml b/test/functional/tools/dbkey_filter_multi_input.xml index 7ecaeaaf943..e26bec1febe 100644 --- a/test/functional/tools/dbkey_filter_multi_input.xml +++ b/test/functional/tools/dbkey_filter_multi_input.xml @@ -1,5 +1,5 @@ - Filter select on dbkey of multiple inputs + Filter select on dbkey of multiple inputs and use of named column > $output; @@ -10,7 +10,7 @@ - + @@ -31,4 +31,4 @@ - \ No newline at end of file + From db4dce21b74b16c100424ee462270bc8558df760 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sat, 17 Aug 2019 01:13:40 +0200 Subject: [PATCH 3/7] added few comments --- lib/galaxy/tools/parameters/dynamic_options.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index 92ff35ac036..5c576c66225 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -179,6 +179,9 @@ class DataMetaFilter(Filter): if not isinstance(ref, HistoryDatasetAssociation) and not is_data_or_data_list: return [] # not a valid dataset + # get the metadata value. for lists (of data sets) and collections + # the meta data value of all elements is determined if its the same + # for all, if different are found the filter returns an empty list if is_data_list: meta_value = None for single_ref in ref: @@ -193,6 +196,8 @@ class DataMetaFilter(Filter): else: meta_value = ref.metadata.get(self.key, None) + # if no meta data value could be determined just return a copy + # of the original options if meta_value is None: return [(disp_name, optval, selected) for disp_name, optval, selected in options] From 96db74d0f47cfc9804bebc83231bc1483a872fa4 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 18 Aug 2019 16:22:06 +0200 Subject: [PATCH 4/7] add tests for static_value and regexp filter --- .../functional/tools/filter_static_regexp.xml | 94 +++++++++++++++++++ test/functional/tools/samples_tool_conf.xml | 1 + 2 files changed, 95 insertions(+) create mode 100644 test/functional/tools/filter_static_regexp.xml diff --git a/test/functional/tools/filter_static_regexp.xml b/test/functional/tools/filter_static_regexp.xml new file mode 100644 index 00000000000..367a16f2484 --- /dev/null +++ b/test/functional/tools/filter_static_regexp.xml @@ -0,0 +1,94 @@ + + Filter by static value and regexp + + echo $index_static > '$output' + echo $index_static_keep >> '$output' + echo $index_regexp >> '$output' + echo $index_regexp_keep >> '$output' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 493582f31ff..d89dc4b335f 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -34,6 +34,7 @@ + From c093cf9fa0572965876f8f66931c63fc13f5f407 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 18 Aug 2019 18:22:03 +0200 Subject: [PATCH 5/7] add regexp to filters in xsd and further work on filter documentation --- lib/galaxy/tool_util/xsd/galaxy.xsd | 42 +++++++++++++++++------------ 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 9bbc11933b7..b9d15996fd0 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -3496,7 +3496,7 @@ is the ``none`` preset. `` tag set - filter out values obtained from a locally stored file (e.g. +```` tag set - modify (e.g. remove, add, sort, ...) the list of values obtained from a locally stored file (e.g. a tool data table) or a dataset in the current history. ### Examples @@ -3599,31 +3599,38 @@ demonstrates splitting up strings into multiple values. - Column targeted by this filter - this -attribute is unused and invalid if ``type`` is ``add_value`` or ``remove_value``. -This can be a column index or a column name. + Column targeted by this filter given as column index or a column name. Invalid if ``type`` is ``add_value`` or ``remove_value``. + @@ -3665,7 +3672,7 @@ only used if ``multiple`` is set to ``true``.]]> If ``true``, keep columns matching the value, if ``false`` discard columns matching the value. Used when ``type`` is -either ``static_value`` or ``param_value``. +either ``static_value``, ``regexp`` or ``param_value``. @@ -3673,7 +3680,7 @@ either ``static_value`` or ``param_value``. Target value of the operations - has slightly different meanings depending on ``type``. For instance when ``type`` is ``add_value`` it is the value to add to the list and when ``type`` is -``static_value`` it is the value compared against. +``static_value`` or ``regexp`` it is the value compared against. @@ -5450,6 +5457,7 @@ and ``bibtex`` are the only supported options. + From ce13eef8055255db73fa76a4e783989811f5f902 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 18 Aug 2019 20:32:38 +0200 Subject: [PATCH 6/7] Completed filter type docs --- lib/galaxy/tool_util/xsd/galaxy.xsd | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index b9d15996fd0..c31d1cc73c2 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -3608,22 +3608,22 @@ Currently the following filters are defined: If no ``column`` is given the metadata value of the referenced input is added to the options list (in this case the corresponding ``options`` tag must not have the ``from_data_table`` or ``from_dataset`` attributes). In both cases the desired metadata is selected by ``key``. -The filter can be inverted in all these cases by setting ``keep`` to true. +The above filters can be inverted by setting ``keep`` to true. * ``add_value``: add an option with a given ``name`` and ``value`` to the options. By default the new option is appended, with ``index`` the insertion position can be specified. * ``remove_value``: remove a value from the options. Either specified explicitly with ``value``, the value of another input specifified with ``ref``, or the metatdata ``key`` of another input ``meta_ref``. -* ``unique_value``: remove options that have duplicate entries in the given ``column`` - -* ``sort_by``: sort options by the entries of a given column. - -* ``multiple_splitter``: split the entries of the referenced of specified ``column``(s) using a ``separator``. -* ``attribute_value_splitter``: +* ``unique_value``: remove options that have duplicate entries in the given ``column``. +* ``sort_by``: sort options by the entries of a given ``column``. +* ``multiple_splitter``: split the entries of the specified ``column``(s) in the referenced file using a ``separator``. Thereby the number of columns is increased. Note that filters that are applyed after one of the latter two filters must not refer to These values are defined in the module [/lib/galaxy/tools/parameters/dynamic_options.py](https://github.com/galaxyproject/galaxy/blob/master/lib/galaxy/tools/parameters/dynamic_options.py) -in the ``filter_types`` dictionary +in the ``filter_types`` dictionary. + + +* ``attribute_value_splitter``: ]]> From 82acc37806a642b62ab311ed64efca1d29d4bb13 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 18 Aug 2019 20:35:12 +0200 Subject: [PATCH 7/7] document attribute_value_filter as deprecated --- lib/galaxy/tool_util/xsd/galaxy.xsd | 3 ++- lib/galaxy/tools/parameters/dynamic_options.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index c31d1cc73c2..da3ba309d06 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -3622,8 +3622,9 @@ These values are defined in the module [/lib/galaxy/tools/parameters/dynamic_options.py](https://github.com/galaxyproject/galaxy/blob/master/lib/galaxy/tools/parameters/dynamic_options.py) in the ``filter_types`` dictionary. +Deprecated filter types: -* ``attribute_value_splitter``: +* ``attribute_value_splitter`` ]]> diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index 5c576c66225..5f449300fac 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -335,6 +335,8 @@ class AttributeValueSplitterFilter(Filter): """ Filters a list of attribute-value pairs to be unique attribute names. + DEPRECATED: just replace with 2 rounds of MultipleSplitterFilter + Type: attribute_value_splitter Required Attributes: