From ee4c5d0817871cff7fe1022d8a8b9b7da50dd68c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 14 Jul 2021 15:24:38 +0200 Subject: [PATCH 1/2] deprecate data manager version Data managers (DM) could define a version for the data manager (``) and for the data manager tool (DMT). The version attribute of the data manager was undocumented and only used seldomly (for IUC in 24 out of 88). If no version was set for the DM it defaults to 0.0.1. Hence in `shed_data_manager_conf.xml` multiple versions of the (DM) with version 0.0.1 are listed. This leads to loading only one DM (I guess the oldest) and a warning like the following on startup: `A data manager has been defined twice: toolshed.g2.bx.psu.edu/repos/galaxyp/data_manager_eggnog_mapper/data_manager/data_manager_eggnog/0.0.1 ` which is easily overlooked. --- doc/source/dev/data_managers.rst | 4 +++- lib/galaxy/tools/data_manager/manager.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/dev/data_managers.rst b/doc/source/dev/data_managers.rst index 6b0737e4d17..7320385f75b 100644 --- a/doc/source/dev/data_managers.rst +++ b/doc/source/dev/data_managers.rst @@ -101,6 +101,8 @@ This tag defines a particular Data Manager. Any number of +---------------+------------+-----------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``id`` | A string* | no | ``id="twobit_builder"`` | Must be unique across all Data Managers; should be lowercase and contain only letters, numbers, and underscores. While technically optional, it is a best-practice to specify this value. When not specified, it will use the id of the underlying Data Manager Tool. | +---------------+------------+-----------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``version`` | A string* | no | ``version="0.0.1"`` | Deprecated with release 21.09. The version of the data manager defaults to the version of the data manager tool | ++---------------+------------+-----------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ The following is an example that contains all of the attributes described above. @@ -616,7 +618,7 @@ the ``extra_files_path`` of ``out_file``. .. code-block:: xml - + fetching data_manager_fetch_genome_all_fasta.py "${out_file}" --dbkey_description ${ dbkey.get_display_text() } diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 25f9f89214b..e26425d10b5 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -140,7 +140,6 @@ class DataManager: self.declared_id = elem.get('id') self.guid = elem.get('guid') path = elem.get('tool_file') - self.version = elem.get('version', self.version) tool_shed_repository = None tool_guid = None @@ -169,6 +168,7 @@ class DataManager: self.name = elem.get('name', self.tool.name) self.description = elem.get('description', self.tool.description) self.undeclared_tables = util.asbool(elem.get('undeclared_tables', self.undeclared_tables)) + self.version = elem.get('version', self.tool.version) for data_table_elem in elem.findall('data_table'): data_table_name = data_table_elem.get("name") From c2a113bc85816129325ae2734f7a24acd69db28d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 14 Jul 2021 17:13:26 +0200 Subject: [PATCH 2/2] add script for fixing shed_data_manager_conf.xml --- scripts/fix_dm_versions.py | 77 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 scripts/fix_dm_versions.py diff --git a/scripts/fix_dm_versions.py b/scripts/fix_dm_versions.py new file mode 100755 index 00000000000..0983e305505 --- /dev/null +++ b/scripts/fix_dm_versions.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +import argparse +import shutil +from datetime import datetime + +from lxml import etree + +desc = """ +Fix shed_data_manager_conf.xml + +Modifies the guid and version attribute of data_manager tags: + +- the version in the guid (text after the last slash) is replaced + by the data manager tool version +- the version attribute is set to the data manager tool version + +By default only data managers with duplicated guid are modified +and version attributes are not added if absent. + +A copy of the original file with a time stamp appended to the name +is created. + +Note, if there are versions of the data manager tool that have the +same version there will still be DMs with duplicated guids. These +need to be corrected manually. +""" + +parser = argparse.ArgumentParser(description=desc) +parser.add_argument('shed_data_manager_conf', metavar='CONFIG_FILE', type=str, + default="config/shed_data_manager_conf.xml", + help='an integer for the accumulator') +parser.add_argument('--all-entries', action='store_true', + help='modify all entries (default only those with duplicated guid)') +parser.add_argument('--add-version', action='store_true', + help='also add version attribute if absent') +parser.add_argument('--dry-run', action='store_true', + help='do not write resulting config file') +args = parser.parse_args() + +with open(args.shed_data_manager_conf) as fh: + tree = etree.parse(args.shed_data_manager_conf) +root = tree.getroot() + +guid_mapping = dict() +for dm in root.iter('data_manager'): + guid = dm.attrib["guid"] + if guid not in guid_mapping: + guid_mapping[guid] = [dm] + else: + guid_mapping[guid].append(dm) + +for guid in guid_mapping: + if len(guid_mapping[guid]) > 1: + print(f'{guid} found {len(guid_mapping[guid])}x') + elif not args.all_entries: + continue + + for dm in guid_mapping[guid]: + tool_version = dm.find("./tool/version") + tool_version = tool_version.text + + new_guid = f"{guid[:guid.rfind('/')]}/{tool_version}" + dm.attrib['guid'] = new_guid + print(f"changing guid: {guid} -> {new_guid}") + if "version" in dm.attrib: + print(f"changing version: {dm.attrib['version']} -> {tool_version}") + dm.attrib['version'] = tool_version + elif args.add_version: + print(f"adding version: {tool_version}") + dm.attrib['version'] = tool_version + +if not args.dry_run: + nfn = args.shed_data_manager_conf + datetime.now().isoformat() + print(f"save copy at {nfn}") + shutil.copyfile(args.shed_data_manager_conf, nfn) + print(f"saving {args.shed_data_manager_conf}") + tree.write(args.shed_data_manager_conf)