diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index ab34acc299f..07d0a9e8d73 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -8,6 +8,7 @@ import shutil import string import tempfile import zipfile +from collections import OrderedDict from inspect import isclass import six @@ -25,7 +26,6 @@ from galaxy.util import ( unicodify ) from galaxy.util.bunch import Bunch -from galaxy.util.odict import odict from galaxy.util.sanitize_html import sanitize_html from . import ( dataproviders, @@ -100,7 +100,7 @@ class Data(object): allow_datatype_change = True # Composite datatypes composite_type = None - composite_files = odict() + composite_files = OrderedDict() primary_file_name = 'index' # A per datatype setting (inherited): max file size (in bytes) for setting optional metadata _max_optional_metadata_filesize = None @@ -116,7 +116,7 @@ class Data(object): object.__init__(self, **kwd) self.supported_display_apps = self.supported_display_apps.copy() self.composite_files = self.composite_files.copy() - self.display_applications = odict() + self.display_applications = OrderedDict() def get_raw_data(self, dataset): """Returns the full data. To stream it open the file_name and read/write as needed""" @@ -546,7 +546,7 @@ class Data(object): return self.display_applications.get(key, default) def get_display_applications_by_dataset(self, dataset, trans): - rval = odict() + rval = OrderedDict() for key, value in self.display_applications.items(): value = value.filter_by_dataset(dataset, trans) if value.links: @@ -666,7 +666,7 @@ class Data(object): @property def writable_files(self, dataset=None): - files = odict() + files = OrderedDict() if self.composite_type != 'auto_primary_file': files[self.primary_file_name] = self.__new_composite_file(self.primary_file_name) for key, value in self.get_composite_files(dataset=dataset).items(): @@ -682,7 +682,7 @@ class Data(object): meta_value = self.metadata_spec[composite_file.substitute_name_with_metadata].default return key % meta_value return key - files = odict() + files = OrderedDict() for key, value in self.composite_files.items(): files[substitute_composite_key(key, value)] = value return files diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index bdf6ff9fac5..036bd1a82d4 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -6,7 +6,7 @@ from __future__ import absolute_import import imp import logging import os -from collections import OrderedDict as odict +from collections import OrderedDict from string import Template from xml.etree.ElementTree import Element @@ -42,7 +42,7 @@ class Registry(object): self.config = config self.datatypes_by_extension = {} self.mimetypes_by_extension = {} - self.datatype_converters = odict() + self.datatype_converters = OrderedDict() # Converters defined in local datatypes_conf.xml self.converters = [] # Converters defined in datatypes_conf.xml included in installed tool shed repositories. @@ -58,7 +58,7 @@ class Registry(object): # tool shed repositories that contain display applications. self.proprietary_display_app_containers = [] # Map a display application id to a display application - self.display_applications = odict() + self.display_applications = OrderedDict() # The following 2 attributes are used in the to_xml_file() # method to persist the current state into an xml file. self.display_path_attr = None @@ -636,7 +636,7 @@ class Registry(object): else: toolbox.register_tool(converter) if source_datatype not in self.datatype_converters: - self.datatype_converters[source_datatype] = odict() + self.datatype_converters[source_datatype] = OrderedDict() self.datatype_converters[source_datatype][target_datatype] = converter if not hasattr(toolbox.app, 'tool_cache') or converter.id in toolbox.app.tool_cache._new_tool_ids: self.log.debug("Loaded converter: %s", converter.id) @@ -874,7 +874,7 @@ class Registry(object): def get_converters_by_datatype(self, ext): """Returns available converters by source type""" if ext not in self._converters_by_datatype: - converters = odict() + converters = OrderedDict() source_datatype = type(self.get_datatype_by_extension(ext)) for ext2, converters_dict in self.datatype_converters.items(): converter_datatype = type(self.get_datatype_by_extension(ext2)) diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index 1e64e9ef2f8..24cb4766696 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -2,12 +2,12 @@ Provides utilities for working with GFF files. """ import copy +from collections import OrderedDict from bx.intervals.io import GenomicInterval, GenomicIntervalReader, MissingFieldError, NiceReaderWrapper, ParseError from bx.tabular.io import Comment, Header from galaxy.util import unicodify -from galaxy.util.odict import odict FASTA_DIRECTIVE = '##FASTA' @@ -427,7 +427,7 @@ def read_unordered_gtf(iterator, strict=False): return fields[0] + '_' + get_transcript_id(fields) # Aggregate intervals by transcript_id and collect comments. - feature_intervals = odict() + feature_intervals = OrderedDict() comments = [] for count, line in enumerate(iterator): if line.startswith('#'): diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py index d670e3faada..f01bc5eca39 100644 --- a/lib/galaxy/job_execution/output_collect.py +++ b/lib/galaxy/job_execution/output_collect.py @@ -4,6 +4,7 @@ import logging import operator import os import re +from collections import OrderedDict from tempfile import NamedTemporaryFile import galaxy.model @@ -27,7 +28,6 @@ from galaxy.tool_util.parser.output_collection_def import ( ToolProvidedMetadataDatasetCollection, ) from galaxy.util import ( - odict, unicodify ) @@ -194,7 +194,7 @@ class JobContext(ModelPersistenceContext): return self.app.tag_handler def find_files(self, output_name, collection, dataset_collectors): - filenames = odict.odict() + filenames = OrderedDict() for discovered_file in discover_files(output_name, self.tool_provided_metadata, dataset_collectors, self.job_working_directory, collection): filenames[discovered_file.path] = discovered_file return filenames @@ -267,7 +267,7 @@ def collect_primary_datasets(job_context, output, input_ext): dataset_collectors = [DEFAULT_DATASET_COLLECTOR] if name in tool.outputs: dataset_collectors = [dataset_collector(description) for description in tool.outputs[name].dataset_collector_descriptions] - filenames = odict.odict() + filenames = OrderedDict() for discovered_file in discover_files(name, job_context.tool_provided_metadata, dataset_collectors, job_working_directory, outdata): filenames[discovered_file.path] = discovered_file for filename_index, (filename, discovered_file) in enumerate(filenames.items()): @@ -284,7 +284,7 @@ def collect_primary_datasets(job_context, output, input_ext): primary_output_assigned = True continue if name not in primary_datasets: - primary_datasets[name] = odict.odict() + primary_datasets[name] = OrderedDict() visible = fields_match.visible ext = fields_match.ext if ext == "input": diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index 3e80738527e..bc214926dbd 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -1,4 +1,5 @@ import logging +from collections import OrderedDict from galaxy import model from galaxy.exceptions import ( @@ -18,7 +19,6 @@ from galaxy.model.dataset_collections.matching import MatchingCollections from galaxy.model.dataset_collections.registry import DATASET_COLLECTION_TYPES_REGISTRY from galaxy.model.dataset_collections.type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY from galaxy.util import ( - odict, validation ) @@ -358,7 +358,7 @@ class DatasetCollectionManager(object): if elements is self.ELEMENTS_UNINITIALIZED: return - new_elements = odict.odict() + new_elements = OrderedDict() for key, element in elements.items(): if isinstance(element, model.DatasetCollection): continue @@ -367,7 +367,7 @@ class DatasetCollectionManager(object): continue # element is a dict with src new_collection and - # and odict of named elements + # and OrderedDict of named elements collection_type = element.get("collection_type", None) sub_elements = element["elements"] collection = self.create_dataset_collection( @@ -381,7 +381,7 @@ class DatasetCollectionManager(object): elements.update(new_elements) def __load_elements(self, trans, element_identifiers, hide_source_items=False, copy_elements=False): - elements = odict.odict() + elements = OrderedDict() for element_identifier in element_identifiers: elements[element_identifier["name"]] = self.__load_element(trans, element_identifier=element_identifier, @@ -477,7 +477,7 @@ class DatasetCollectionManager(object): def _build_elements_from_rule_data(self, collection_type_description, rule_set, data, sources, handle_dataset): identifier_columns = rule_set.identifier_columns - elements = odict.odict() + elements = OrderedDict() for data_index, row_data in enumerate(data): # For each row, find place in depth for this element. collection_type_at_depth = collection_type_description @@ -508,7 +508,7 @@ class DatasetCollectionManager(object): sub_collection = {} sub_collection["src"] = "new_collection" sub_collection["collection_type"] = collection_type_at_depth.collection_type - sub_collection["elements"] = odict.odict() + sub_collection["elements"] = OrderedDict() elements_at_depth[identifier] = sub_collection elements_at_depth = sub_collection["elements"] diff --git a/lib/galaxy/model/dataset_collections/builder.py b/lib/galaxy/model/dataset_collections/builder.py index c24af953f94..805e68adff5 100644 --- a/lib/galaxy/model/dataset_collections/builder.py +++ b/lib/galaxy/model/dataset_collections/builder.py @@ -1,5 +1,6 @@ +from collections import OrderedDict + from galaxy import model -from galaxy.util.odict import odict from .type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY @@ -34,7 +35,7 @@ class CollectionBuilder(object): def __init__(self, collection_type_description): self._collection_type_description = collection_type_description - self._current_elements = odict() + self._current_elements = OrderedDict() def replace_elements_in_collection(self, template_collection, replacement_dict): self._current_elements = self._replace_elements_in_collection( @@ -43,7 +44,7 @@ class CollectionBuilder(object): ) def _replace_elements_in_collection(self, template_collection, replacement_dict): - elements = odict() + elements = OrderedDict() for element in template_collection.elements: if element.is_collection: collection_builder = CollectionBuilder( @@ -77,7 +78,7 @@ class CollectionBuilder(object): def build_elements(self): elements = self._current_elements if self._nested_collection: - new_elements = odict() + new_elements = OrderedDict() for identifier, element in elements.items(): new_elements[identifier] = element.build() elements = new_elements diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index 3b5506e143a..9fdc474c19e 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -11,6 +11,7 @@ import shutil import sys import tempfile import weakref +from collections import OrderedDict from os.path import abspath from six import string_types @@ -26,7 +27,6 @@ from galaxy.util import ( ) from galaxy.util.json import safe_dumps from galaxy.util.object_wrapper import sanitize_lists_to_string -from galaxy.util.odict import odict log = logging.getLogger(__name__) @@ -214,7 +214,7 @@ class MetadataCollection(object): return None -class MetadataSpecCollection(odict): +class MetadataSpecCollection(OrderedDict): """ A simple extension of dict which allows cleaner access to items and allows the values to be iterated over directly as if it were a @@ -223,7 +223,7 @@ class MetadataSpecCollection(odict): """ def __init__(self, dict=None): - odict.__init__(self, dict=None) + OrderedDict.__init__(self, dict=None) def append(self, item): self[item.name] = item diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 2f5964d4ed9..9a048a76a00 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -8,7 +8,10 @@ corresponding to files in other contexts. import abc import logging import os -from collections import namedtuple +from collections import ( + namedtuple, + OrderedDict +) import six @@ -19,8 +22,7 @@ from galaxy.exceptions import ( ) from galaxy.model.dataset_collections import builder from galaxy.util import ( - ExecutionTimer, - odict + ExecutionTimer ) from galaxy.util.hash_util import HASH_NAME_MAP @@ -436,7 +438,7 @@ def persist_target_to_export_store(target_dict, export_store, object_store, work def persist_elements_to_hdca(model_persistence_context, elements, hdca, collector=None): - filenames = odict.odict() + filenames = OrderedDict() def add_to_discovered_files(elements, parent_identifiers=[]): for element in elements: diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index a10b56e97ab..ce8f2a579b5 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -11,6 +11,7 @@ import random import shutil import threading import time +from collections import OrderedDict from xml.etree import ElementTree import yaml @@ -26,7 +27,6 @@ from galaxy.util import ( umask_fix_perms, ) from galaxy.util.bunch import Bunch -from galaxy.util.odict import odict from galaxy.util.path import ( safe_makedirs, safe_relpath, @@ -804,7 +804,7 @@ class HierarchicalObjectStore(NestedObjectStore): """The default contructor. Extends `NestedObjectStore`.""" super(HierarchicalObjectStore, self).__init__(config, config_dict) - backends = odict() + backends = OrderedDict() for order, backend_def in enumerate(config_dict["backends"]): backends[order] = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon) diff --git a/lib/galaxy/openid/providers.py b/lib/galaxy/openid/providers.py index 1cd17795d3e..61f65520601 100644 --- a/lib/galaxy/openid/providers.py +++ b/lib/galaxy/openid/providers.py @@ -3,11 +3,11 @@ Contains OpenID provider functionality """ import logging import os +from collections import OrderedDict import six from galaxy.util import parse_xml, string_as_bool -from galaxy.util.odict import odict log = logging.getLogger(__name__) @@ -116,7 +116,7 @@ class OpenIDProviders(object): @classmethod def from_elem(cls, xml_root): oid_elem = xml_root - providers = odict() + providers = OrderedDict() for elem in oid_elem.findall('provider'): try: provider = OpenIDProvider.from_file(os.path.join('lib/galaxy/openid', elem.get('file'))) @@ -130,7 +130,7 @@ class OpenIDProviders(object): if providers: self.providers = providers else: - self.providers = odict() + self.providers = OrderedDict() self._banned_identifiers = [provider.op_endpoint_url for provider in self.providers.values() if provider.never_associate_with_user] def __iter__(self): diff --git a/lib/galaxy/tool_util/cwl/parser.py b/lib/galaxy/tool_util/cwl/parser.py index 9d7de0d0bf4..6651855635d 100644 --- a/lib/galaxy/tool_util/cwl/parser.py +++ b/lib/galaxy/tool_util/cwl/parser.py @@ -11,13 +11,13 @@ import logging import os import pickle from abc import ABCMeta, abstractmethod +from collections import OrderedDict import six from galaxy.exceptions import MessageException from galaxy.util import listify, safe_makedirs from galaxy.util.bunch import Bunch -from galaxy.util.odict import odict from .cwltool_deps import ( ensure_cwltool_available, pathmapper, @@ -1091,7 +1091,7 @@ class ConditionalInstance(object): name=self.name, type=INPUT_TYPE.CONDITIONAL, test=self.case.to_dict(), - when=odict(), + when=OrderedDict(), ) for value, block in self.whens: as_dict["when"][value] = [i.to_dict() for i in block] diff --git a/lib/galaxy/tool_util/parser/cwl.py b/lib/galaxy/tool_util/parser/cwl.py index bdec03eb641..7e1f2199918 100644 --- a/lib/galaxy/tool_util/parser/cwl.py +++ b/lib/galaxy/tool_util/parser/cwl.py @@ -1,9 +1,9 @@ import logging import os +from collections import OrderedDict from galaxy.tool_util.cwl import tool_proxy from galaxy.tool_util.deps import requirements -from galaxy.util.odict import odict from .error_level import StdioErrorLevel from .interface import ( PageSource, @@ -104,14 +104,14 @@ class CwlToolSource(ToolSource): def parse_outputs(self, tool): output_instances = self.tool_proxy.output_instances() - outputs = odict() + outputs = OrderedDict() output_defs = [] for output_instance in output_instances: output_defs.append(self._parse_output(tool, output_instance)) # TODO: parse outputs collections for output_def in output_defs: outputs[output_def.name] = output_def - return outputs, odict() + return outputs, OrderedDict() def _parse_output(self, tool, output_instance): name = output_instance.name diff --git a/lib/galaxy/tool_util/parser/factory.py b/lib/galaxy/tool_util/parser/factory.py index 63a65fea3c8..c17fc116515 100644 --- a/lib/galaxy/tool_util/parser/factory.py +++ b/lib/galaxy/tool_util/parser/factory.py @@ -2,11 +2,11 @@ from __future__ import absolute_import import logging +from collections import OrderedDict import yaml from galaxy.tool_util.loader import load_tool_with_refereces -from galaxy.util.odict import odict from .cwl import CwlToolSource from .interface import InputSource from .xml import XmlInputSource, XmlToolSource @@ -54,7 +54,7 @@ def ordered_load(stream): def construct_mapping(loader, node): loader.flatten_mapping(node) - return odict(loader.construct_pairs(node)) + return OrderedDict(loader.construct_pairs(node)) OrderedLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py index 9431f97be77..392babefae4 100644 --- a/lib/galaxy/tool_util/parser/output_objects.py +++ b/lib/galaxy/tool_util/parser/output_objects.py @@ -1,5 +1,6 @@ +from collections import OrderedDict + from galaxy.util.dictifiable import Dictifiable -from galaxy.util.odict import odict from .output_actions import ToolOutputActionGroup from .output_collection_def import dataset_collector_descriptions_from_output_dict @@ -152,7 +153,7 @@ class ToolOutputCollection(ToolOutputBase): self.collection = True self.default_format = default_format self.structure = structure - self.outputs = odict() + self.outputs = OrderedDict() self.inherit_format = inherit_format self.inherit_metadata = inherit_metadata diff --git a/lib/galaxy/tool_util/parser/util.py b/lib/galaxy/tool_util/parser/util.py index 38fb467d7f9..863621af9b1 100644 --- a/lib/galaxy/tool_util/parser/util.py +++ b/lib/galaxy/tool_util/parser/util.py @@ -1,11 +1,12 @@ -from galaxy.util.odict import odict +from collections import OrderedDict + from .error_level import StdioErrorLevel from .interface import ToolStdioExitCode from .interface import ToolStdioRegex def is_dict(item): - return isinstance(item, dict) or isinstance(item, odict) + return isinstance(item, dict) or isinstance(item, OrderedDict) def error_on_exit_code(out_of_memory_exit_code=None): diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index ffd78281903..94c5a076f3d 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -3,13 +3,13 @@ import re import sys import traceback import uuid +from collections import OrderedDict from math import isinf import packaging.version from galaxy.tool_util.deps import requirements from galaxy.util import string_as_bool, xml_text, xml_to_string -from galaxy.util.odict import odict from .error_level import StdioErrorLevel from .interface import ( InputSource, @@ -261,12 +261,12 @@ class XmlToolSource(ToolSource): def parse_outputs(self, tool): out_elem = self.root.find("outputs") - outputs = odict() - output_collections = odict() + outputs = OrderedDict() + output_collections = OrderedDict() if out_elem is None: return outputs, output_collections - data_dict = odict() + data_dict = OrderedDict() def _parse(data_elem, **kwds): output_def = self._parse_output(data_elem, tool, **kwds) diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index 8790c0419ea..5133b726d8c 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -1,7 +1,8 @@ +from collections import OrderedDict + import packaging.version from galaxy.tool_util.deps import requirements -from galaxy.util.odict import odict from .interface import InputSource from .interface import PageSource from .interface import PagesSource @@ -105,10 +106,10 @@ class YamlToolSource(ToolSource): else: message = "Unknown output_type [%s] encountered." % output_type raise Exception(message) - outputs = odict() + outputs = OrderedDict() for output in output_defs: outputs[output.name] = output - output_collections = odict() + output_collections = OrderedDict() for output in output_collection_defs: output_collections[output.name] = output diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 57b95d0e247..8ab64328799 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -91,7 +91,6 @@ from galaxy.util.dictifiable import Dictifiable from galaxy.util.expressions import ExpressionContext from galaxy.util.form_builder import SelectField from galaxy.util.json import safe_loads -from galaxy.util.odict import odict from galaxy.util.rules_dsl import RuleSet from galaxy.util.template import fill_template from galaxy.version import VERSION_MAJOR @@ -422,7 +421,7 @@ class Tool(Dictifiable): self.repository_id = repository_id self._allow_code_files = allow_code_files # setup initial attribute values - self.inputs = odict() + self.inputs = OrderedDict() self.stdio_exit_codes = list() self.stdio_regexes = list() self.inputs_by_page = list() @@ -1084,7 +1083,7 @@ class Tool(Dictifiable): groups (repeat, conditional) or param elements. Groups will be parsed recursively. """ - rval = odict() + rval = OrderedDict() context = ExpressionContext(rval, context) for input_source in page_source.parse_input_sources(): # Repeat group @@ -1125,7 +1124,7 @@ class Tool(Dictifiable): page_source = XmlPageSource(ElementTree.XML("%s" % case_inputs)) case.inputs = self.parse_input_elem(page_source, enctypes, context) else: - case.inputs = odict() + case.inputs = OrderedDict() group.cases.append(case) else: # Should have one child "input" which determines the case @@ -1153,7 +1152,7 @@ class Tool(Dictifiable): (self.id, group.name, group.test_param.name, unspecified_case)) case = ConditionalWhen() case.value = unspecified_case - case.inputs = odict() + case.inputs = OrderedDict() group.cases.append(case) rval[group.name] = group elif input_type == "section": @@ -1494,7 +1493,7 @@ class Tool(Dictifiable): log.exception('Exception caught while attempting tool execution:') message = 'Error executing tool: %s' % unicodify(e) return False, message - if isinstance(out_data, odict): + if isinstance(out_data, OrderedDict): return job, list(out_data.items()) else: if isinstance(out_data, string_types): @@ -2567,7 +2566,7 @@ class DatabaseOperationTool(Tool): return self._outputs_dict() def _outputs_dict(self): - return odict() + return OrderedDict() class UnzipCollectionTool(DatabaseOperationTool): @@ -2599,7 +2598,7 @@ class ZipCollectionTool(DatabaseOperationTool): reverse_o = incoming["input_reverse"] forward, reverse = forward_o.copy(), reverse_o.copy() - new_elements = odict() + new_elements = OrderedDict() new_elements["forward"] = forward new_elements["reverse"] = reverse self._add_datasets_to_history(history, [forward, reverse]) @@ -2612,7 +2611,7 @@ class BuildListCollectionTool(DatabaseOperationTool): tool_type = 'build_list' def produce_outputs(self, trans, out_data, output_collections, incoming, history, tags=None): - new_elements = odict() + new_elements = OrderedDict() for i, incoming_repeat in enumerate(incoming["datasets"]): new_dataset = incoming_repeat["input"].copy(copy_tags=tags) @@ -2672,7 +2671,7 @@ class MergeCollectionTool(DatabaseOperationTool): if dupl_actions in ['suffix_conflict', 'suffix_every', 'suffix_conflict_rest']: suffix_pattern = advanced['conflict']['suffix_pattern'] - new_element_structure = odict() + new_element_structure = OrderedDict() # Which inputs does the identifier appear in. identifiers_map = {} @@ -2724,7 +2723,7 @@ class MergeCollectionTool(DatabaseOperationTool): new_element_structure[effective_identifer] = element # Don't copy until we know everything is fine and we have the structure of the list ready to go. - new_elements = odict() + new_elements = OrderedDict() for key, value in new_element_structure.items(): if getattr(value, "history_content_type", None) == "dataset": copied_value = value.copy(force_flush=False) @@ -2741,7 +2740,7 @@ class MergeCollectionTool(DatabaseOperationTool): class FilterDatasetsTool(DatabaseOperationTool): def _get_new_elements(self, history, elements_to_copy): - new_elements = odict() + new_elements = OrderedDict() for dce in elements_to_copy: element_identifier = dce.element_identifier if getattr(dce.element_object, "history_content_type", None) == "dataset": @@ -2809,7 +2808,7 @@ class FlattenTool(DatabaseOperationTool): def produce_outputs(self, trans, out_data, output_collections, incoming, history, **kwds): hdca = incoming["input"] join_identifier = incoming["join_identifier"] - new_elements = odict() + new_elements = OrderedDict() copied_datasets = [] def add_elements(collection, prefix=""): @@ -2837,7 +2836,7 @@ class SortTool(DatabaseOperationTool): def produce_outputs(self, trans, out_data, output_collections, incoming, history, **kwds): hdca = incoming["input"] sorttype = incoming["sort_type"]["sort_type"] - new_elements = odict() + new_elements = OrderedDict() elements = hdca.collection.elements presort_elements = [] if sorttype == 'alpha': @@ -2882,7 +2881,7 @@ class RelabelFromFileTool(DatabaseOperationTool): how_type = incoming["how"]["how_select"] new_labels_dataset_assoc = incoming["how"]["labels"] strict = string_as_bool(incoming["how"]["strict"]) - new_elements = odict() + new_elements = OrderedDict() def add_copied_value_to_new_elements(new_label, dce_object): new_label = new_label.strip() @@ -2954,7 +2953,7 @@ class TagFromFileTool(DatabaseOperationTool): hdca = incoming["input"] how = incoming['how'] new_tags_dataset_assoc = incoming["tags"] - new_elements = odict() + new_elements = OrderedDict() tags_manager = GalaxyTagHandler(trans.app.model.context) new_datasets = [] @@ -3015,8 +3014,8 @@ class FilterFromFileTool(DatabaseOperationTool): hdca = incoming["input"] how_filter = incoming["how"]["how_filter"] filter_dataset_assoc = incoming["how"]["filter_source"] - filtered_elements = odict() - discarded_elements = odict() + filtered_elements = OrderedDict() + discarded_elements = OrderedDict() filtered_path = filter_dataset_assoc.file_name filtered_identifiers_raw = open(filtered_path, "r").readlines(1024 * 1000000) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 57693b23eb3..b6b6e409d5e 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -2,6 +2,7 @@ import json import logging import os import re +from collections import OrderedDict from json import dumps from six import string_types @@ -16,7 +17,6 @@ from galaxy.tools.parameters import update_dataset_ids from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue from galaxy.tools.parameters.wrapped import WrappedParameters from galaxy.util import ExecutionTimer -from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.web import url_for @@ -68,7 +68,7 @@ class DefaultToolAction(object): """ if current_user_roles is None: current_user_roles = trans.get_current_user_roles() - input_datasets = odict() + input_datasets = OrderedDict() all_permissions = {} def record_permission(action, role_id): @@ -337,7 +337,7 @@ class DefaultToolAction(object): # wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed wrapped_params = self._wrapped_params(trans, tool, incoming, inp_data) - out_data = odict() + out_data = OrderedDict() input_collections = dict((k, v[0][0]) for k, v in inp_dataset_collections.items()) output_collections = OutputCollections( trans, @@ -824,7 +824,7 @@ class OutputCollections(object): # We don't care about the repeat index, we just need to find the correct DataCollectionToolParameter else: key = group - if isinstance(data_param, odict): + if isinstance(data_param, OrderedDict): data_param = data_param.get(key) else: data_param = data_param.inputs.get(key) diff --git a/lib/galaxy/tools/actions/history_imp_exp.py b/lib/galaxy/tools/actions/history_imp_exp.py index 02f6b048cf4..8891909efa8 100644 --- a/lib/galaxy/tools/actions/history_imp_exp.py +++ b/lib/galaxy/tools/actions/history_imp_exp.py @@ -1,10 +1,10 @@ import logging import os import tempfile +from collections import OrderedDict from galaxy.tools.actions import ToolAction from galaxy.tools.imp_exp import JobExportHistoryArchiveWrapper -from galaxy.util.odict import odict log = logging.getLogger(__name__) @@ -62,7 +62,7 @@ class ImportHistoryToolAction(ToolAction): trans.app.job_manager.enqueue(job, tool=tool) trans.log_event("Added import history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) - return job, odict() + return job, OrderedDict() class ExportHistoryToolAction(ToolAction): @@ -141,4 +141,4 @@ class ExportHistoryToolAction(ToolAction): trans.app.job_manager.enqueue(job, tool=tool) trans.log_event("Added export history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) - return job, odict() + return job, OrderedDict() diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index cfb80622219..5b6ea01f19e 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -1,10 +1,10 @@ import logging import os +from collections import OrderedDict from json import dumps from galaxy.job_execution.datasets import DatasetPath from galaxy.metadata import get_metadata_compute_strategy -from galaxy.util.odict import odict from . import ToolAction log = logging.getLogger(__name__) @@ -118,4 +118,4 @@ class SetMetadataToolAction(ToolAction): # clear e.g. converted files dataset.datatype.before_setting_metadata(dataset) - return job, odict() + return job, OrderedDict() diff --git a/lib/galaxy/tools/actions/model_operations.py b/lib/galaxy/tools/actions/model_operations.py index 6648b384a1d..da815f3dd22 100644 --- a/lib/galaxy/tools/actions/model_operations.py +++ b/lib/galaxy/tools/actions/model_operations.py @@ -1,11 +1,11 @@ import logging +from collections import OrderedDict from galaxy.tools.actions import ( DefaultToolAction, OutputCollections, ToolExecutionCache, ) -from galaxy.util.odict import odict log = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class ModelOperationToolAction(DefaultToolAction): # wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed wrapped_params = self._wrapped_params(trans, tool, incoming) - out_data = odict() + out_data = OrderedDict() input_collections = dict((k, v[0][0]) for k, v in inp_dataset_collections.items()) output_collections = OutputCollections( trans, diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index d3b361b3b0f..6ded968e31a 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -5,6 +5,7 @@ import shlex import socket import subprocess import tempfile +from collections import OrderedDict from json import dump, dumps from six import StringIO @@ -16,7 +17,6 @@ from galaxy import datatypes, util from galaxy.exceptions import ConfigDoesNotAllowException, ObjectInvalid from galaxy.model import tags from galaxy.util import unicodify -from galaxy.util.odict import odict log = logging.getLogger(__name__) @@ -442,7 +442,7 @@ def create_job(trans, params, tool, json_file_path, outputs, folder=None, histor # Queue the job for execution trans.app.job_manager.enqueue(job, tool=tool) trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) - output = odict() + output = OrderedDict() for i, v in enumerate(outputs): if not hasattr(output_object, "collection_type"): output['output%i' % i] = v diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 4fc77b3e948..0149a444b1c 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -14,6 +14,7 @@ import os.path import re import string import time +from collections import OrderedDict from glob import glob from tempfile import NamedTemporaryFile from xml.etree import ElementTree @@ -22,7 +23,6 @@ import requests from galaxy import util from galaxy.util.dictifiable import Dictifiable -from galaxy.util.odict import odict from galaxy.util.renamed_temporary_file import RenamedTemporaryFile log = logging.getLogger(__name__) @@ -251,7 +251,7 @@ class ToolDataTable(object): self.empty_field_values = {} self.allow_duplicate_entries = util.asbool(config_element.get('allow_duplicate_entries', True)) self.here = filename and os.path.dirname(filename) - self.filenames = odict() + self.filenames = OrderedDict() self.tool_data_path = tool_data_path self.tool_data_path_files = tool_data_path_files self.missing_index_file = None diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 9e6f2949203..b2db2349d81 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -2,6 +2,7 @@ import errno import json import logging import os +from collections import OrderedDict from six import string_types @@ -10,7 +11,6 @@ from galaxy.queue_worker import ( send_control_task ) from galaxy.tools.data import TabularToolDataTable -from galaxy.util.odict import odict from galaxy.util.template import fill_template from tool_shed.util import ( common_util, @@ -27,8 +27,8 @@ DEFAULT_VALUE_TRANSLATION_TYPE = 'template' class DataManagers(object): def __init__(self, app, xml_filename=None): self.app = app - self.data_managers = odict() - self.managed_data_tables = odict() + self.data_managers = OrderedDict() + self.managed_data_tables = OrderedDict() self.tool_path = None self._reload_count = 0 self.filename = xml_filename or self.app.config.data_manager_config_file @@ -131,7 +131,7 @@ class DataManager(object): self.version = self.DEFAULT_VERSION self.guid = None self.tool = None - self.data_tables = odict() + self.data_tables = OrderedDict() self.output_ref_by_data_table = {} self.move_by_data_table_column = {} self.value_translation_by_data_table_column = {} @@ -213,7 +213,7 @@ class DataManager(object): data_table_name = data_table_elem.get("name") assert data_table_name is not None, "A name is required for a data table entry" if data_table_name not in self.data_tables: - self.data_tables[data_table_name] = odict() + self.data_tables[data_table_name] = OrderedDict() output_elem = data_table_elem.find('output') if output_elem is not None: for column_elem in output_elem.findall('column'): diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index 00ad022df89..7b16a7a61df 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -30,7 +30,7 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre >>> from xml.etree.ElementTree import XML >>> from galaxy.util.bunch import Bunch - >>> from galaxy.util.odict import odict + >>> from collections import OrderedDict >>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter >>> from galaxy.tools.parameters.grouping import Repeat >>> a = TextToolParameter(None, XML('')) @@ -44,9 +44,9 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre >>> i = TextToolParameter(None, XML('')) >>> j = TextToolParameter(None, XML('')) >>> b.name = b.title = 'b' - >>> b.inputs = odict([ ('c', c), ('d', d) ]) + >>> b.inputs = OrderedDict([ ('c', c), ('d', d) ]) >>> d.name = d.title = 'd' - >>> d.inputs = odict([ ('e', e), ('f', f) ]) + >>> d.inputs = OrderedDict([ ('e', e), ('f', f) ]) >>> f.test_param = g >>> f.name = 'f' >>> f.cases = [Bunch(value='true', inputs= {'h': h}), Bunch(value='false', inputs= { 'i': i })] @@ -55,8 +55,8 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre ... print('name=%s, prefix=%s, prefixed_name=%s, prefixed_label=%s, value=%s' % (input.name, prefix, prefixed_name, prefixed_label, value)) ... if error: ... print(error) - >>> inputs = odict([('a', a),('b', b)]) - >>> nested = odict([('a', 1), ('b', [odict([('c', 3), ('d', [odict([ ('e', 5), ('f', odict([ ('g', True), ('h', 7)]))])])])])]) + >>> inputs = OrderedDict([('a', a),('b', b)]) + >>> nested = OrderedDict([('a', 1), ('b', [OrderedDict([('c', 3), ('d', [OrderedDict([ ('e', 5), ('f', OrderedDict([ ('g', True), ('h', 7)]))])])])])]) >>> visit_input_values(inputs, nested, visitor) name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1 name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3 @@ -104,7 +104,7 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre No value found for 'b 1 > d 1 > j'. >>> # Other parameters are missing in state - >>> nested = odict([('b', [odict([( 'd', [odict([('f', odict([('g', True), ('h', 7)]))])])])])]) + >>> nested = OrderedDict([('b', [OrderedDict([( 'd', [OrderedDict([('f', OrderedDict([('g', True), ('h', 7)]))])])])])]) >>> visit_input_values(inputs, nested, visitor) name=a, prefix=, prefixed_name=a, prefixed_label=a, value=None No value found for 'a'. @@ -271,7 +271,7 @@ def populate_state(request_context, inputs, incoming, state, errors={}, prefix=' Populates nested state dict from incoming parameter values. >>> from xml.etree.ElementTree import XML >>> from galaxy.util.bunch import Bunch - >>> from galaxy.util.odict import odict + >>> from collections import OrderedDict >>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter >>> from galaxy.tools.parameters.grouping import Repeat >>> trans = Bunch(workflow_building_mode=False) @@ -289,15 +289,15 @@ def populate_state(request_context, inputs, incoming, state, errors={}, prefix=' >>> h = TextToolParameter(None, XML('')) >>> i = TextToolParameter(None, XML('')) >>> b.name = 'b' - >>> b.inputs = odict([('c', c), ('d', d)]) + >>> b.inputs = OrderedDict([('c', c), ('d', d)]) >>> d.name = 'd' - >>> d.inputs = odict([('e', e), ('f', f)]) + >>> d.inputs = OrderedDict([('e', e), ('f', f)]) >>> f.test_param = g >>> f.name = 'f' >>> f.cases = [Bunch(value='true', inputs= { 'h': h }), Bunch(value='false', inputs= { 'i': i })] - >>> inputs = odict([('a',a),('b',b)]) - >>> flat = odict([('a', 1), ('b_0|c', 2), ('b_0|d_0|e', 3), ('b_0|d_0|f|h', 4), ('b_0|d_0|f|g', True)]) - >>> state = odict() + >>> inputs = OrderedDict([('a',a),('b',b)]) + >>> flat = OrderedDict([('a', 1), ('b_0|c', 2), ('b_0|d_0|e', 3), ('b_0|d_0|f|h', 4), ('b_0|d_0|f|g', True)]) + >>> state = OrderedDict() >>> populate_state(trans, inputs, flat, state, check=False) >>> print(state['a']) 1 diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index aaa01dc489a..e41ac9f426a 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -3,7 +3,10 @@ import logging import os import string import time -from collections import namedtuple +from collections import ( + namedtuple, + OrderedDict +) from errno import ENOENT from xml.etree.ElementTree import ParseError @@ -23,7 +26,6 @@ from galaxy.util import ( ) from galaxy.util.bunch import Bunch from galaxy.util.dictifiable import Dictifiable -from galaxy.util.odict import odict from .filters import FilterFactory from .integrated_panel import ManagesIntegratedToolPanelMixin from .lineages import LineageMap @@ -82,7 +84,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): # In-memory dictionary that defines the layout of the tool panel. self._tool_panel = ToolPanelElements() self._index = 0 - self.data_manager_tools = odict() + self.data_manager_tools = OrderedDict() self._lineage_map = LineageMap(app) # Sets self._integrated_tool_panel and self._integrated_tool_panel_config_has_contents self._init_integrated_tool_panel(app.config) diff --git a/lib/galaxy/tools/toolbox/panel.py b/lib/galaxy/tools/toolbox/panel.py index 30eceae64b6..4f6b9534cb6 100644 --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -1,10 +1,10 @@ from abc import abstractmethod +from collections import OrderedDict from six import iteritems from galaxy.util import bunch from galaxy.util.dictifiable import Dictifiable -from galaxy.util.odict import odict from .parser import ensure_tool_conf_item @@ -112,7 +112,7 @@ class ToolSectionLabel(Dictifiable): return super(ToolSectionLabel, self).to_dict() -class ToolPanelElements(odict, HasPanelItems): +class ToolPanelElements(OrderedDict, HasPanelItems): """ Represents an ordered dictionary of tool entries - abstraction used both by tool panel itself (normal and integrated) and its sections. """ diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py index 9952f301308..b5d9753fb03 100644 --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -1,5 +1,6 @@ import logging import tempfile +from collections import OrderedDict from functools import total_ordering from six import string_types, text_type @@ -7,7 +8,6 @@ from six.moves import shlex_quote from galaxy import exceptions from galaxy.model.none_like import NoneDataset -from galaxy.util import odict from galaxy.util.object_wrapper import wrap_with_safe_string log = logging.getLogger(__name__) @@ -419,7 +419,7 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets): self.collection = collection elements = collection.elements - element_instances = odict.odict() + element_instances = OrderedDict() element_instance_list = [] for dataset_collection_element in elements: diff --git a/lib/galaxy/util/odict.py b/lib/galaxy/util/odict.py deleted file mode 100644 index 3f123d2ecf0..00000000000 --- a/lib/galaxy/util/odict.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Ordered dictionary implementation. -""" - -from six.moves import UserDict -dict_alias = dict - - -class odict(UserDict): - """ - http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/107747 - - This dictionary class extends UserDict to record the order in which items are - added. Calling keys(), values(), items(), etc. will return results in this - order. - """ - - def __init__(self, dict=None): - item = dict - self._keys = [] - if isinstance(item, dict_alias): - UserDict.__init__(self, item) - else: - UserDict.__init__(self, None) - if isinstance(item, list): - for (key, value) in item: - self[key] = value - - def __delitem__(self, key): - UserDict.__delitem__(self, key) - self._keys.remove(key) - - def __setitem__(self, key, item): - UserDict.__setitem__(self, key, item) - if key not in self._keys: - self._keys.append(key) - - def clear(self): - UserDict.clear(self) - self._keys = [] - - def copy(self): - new = odict() - new.update(self) - return new - - def items(self): - return zip(self._keys, self.values()) - - def keys(self): - return self._keys[:] - - def popitem(self): - try: - key = self._keys[-1] - except IndexError: - raise KeyError('dictionary is empty') - val = self[key] - del self[key] - return (key, val) - - def setdefault(self, key, failobj=None): - if key not in self._keys: - self._keys.append(key) - return UserDict.setdefault(self, key, failobj) - - def update(self, dict): - for (key, val) in dict.items(): - self.__setitem__(key, val) - - def values(self): - return map(self.get, self._keys) - - def iterkeys(self): - return iter(self._keys) - - def itervalues(self): - for key in self._keys: - yield self.get(key) - - def iteritems(self): - for key in self._keys: - yield key, self.get(key) - - def __iter__(self): - for key in self._keys: - yield key - - def reverse(self): - self._keys.reverse() - - def insert(self, index, key, item): - if key not in self._keys: - self._keys.insert(index, key) - UserDict.__setitem__(self, key, item) diff --git a/lib/galaxy/util/simplegraph.py b/lib/galaxy/util/simplegraph.py index 40a4a3fb363..e185f9e5e95 100644 --- a/lib/galaxy/util/simplegraph.py +++ b/lib/galaxy/util/simplegraph.py @@ -3,7 +3,7 @@ Fencepost-simple graph structure implementation. """ # Currently (2013.7.12) only used in easing the parsing of graph datatype data. -from galaxy.util.odict import odict +from collections import OrderedDict class SimpleGraphNode(object): @@ -58,7 +58,7 @@ class SimpleGraph(object): def __init__(self, nodes=None, edges=None): # use an odict so that edge indeces actually match the final node list indeces - self.nodes = nodes or odict() + self.nodes = nodes or OrderedDict() self.edges = edges or [] def add_node(self, node_id, **data): diff --git a/lib/galaxy/util/topsort.py b/lib/galaxy/util/topsort.py index 82de81eb26e..9a40e7d7a8e 100644 --- a/lib/galaxy/util/topsort.py +++ b/lib/galaxy/util/topsort.py @@ -36,7 +36,7 @@ then CycleError is raised, and the exception object supports many methods to help analyze and break the cycles. This requires a good deal more code than topsort itself! """ -from galaxy.util.odict import odict as OrderedDict +from collections import OrderedDict as OrderedDict class CycleError(Exception): diff --git a/lib/galaxy/visualization/plugins/registry.py b/lib/galaxy/visualization/plugins/registry.py index 95983ab7c7f..ed7bb01f751 100644 --- a/lib/galaxy/visualization/plugins/registry.py +++ b/lib/galaxy/visualization/plugins/registry.py @@ -7,11 +7,11 @@ Lower level of visualization framework which does three main things: import logging import os import weakref +from collections import OrderedDict from galaxy.exceptions import ObjectNotFound from galaxy.util import ( config_directories_from_setting, - odict, parse_xml ) from galaxy.visualization.plugins import ( @@ -66,7 +66,7 @@ class VisualizationsRegistry(object): self.additional_template_paths = [] self.directories = [] self.skip_bad_plugins = skip_bad_plugins - self.plugins = odict.odict() + self.plugins = OrderedDict() self.directories = config_directories_from_setting(directories_setting, app.config.root) self._load_configuration() self._load_plugins() @@ -107,7 +107,7 @@ class VisualizationsRegistry(object): """ Search ``self.directories`` for potential plugins, load them, and cache in ``self.plugins``. - :rtype: odict + :rtype: OrderedDict :returns: ``self.plugins`` """ for plugin_path in self._find_plugins(): diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index 6031898535b..8923196b5e4 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -1,5 +1,6 @@ import logging import math +from collections import OrderedDict from json import dumps, loads from markupsafe import escape @@ -8,7 +9,6 @@ from sqlalchemy.sql.expression import and_, false, func, null, or_, true from galaxy.model.item_attrs import get_foreign_key, UsesAnnotations, UsesItemRatings from galaxy.util import restore_text, sanitize_text, unicodify -from galaxy.util.odict import odict from galaxy.web.framework import decorators, url_for @@ -873,7 +873,7 @@ class SharingStatusColumn(GridColumn): def get_accepted_filters(self): """ Returns a list of accepted filters for this column. """ - accepted_filter_labels_and_vals = odict() + accepted_filter_labels_and_vals = OrderedDict() accepted_filter_labels_and_vals["private"] = "private" accepted_filter_labels_and_vals["shared"] = "shared" accepted_filter_labels_and_vals["accessible"] = "accessible" diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index f920b44b3be..94c7693cb29 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -1,5 +1,6 @@ import logging import os +from collections import OrderedDict from json import dumps, loads import galaxy.queue_worker @@ -7,7 +8,6 @@ from galaxy import exceptions, managers, util, web from galaxy.managers.collections_util import dictify_dataset_collection_instance from galaxy.tools import global_tool_errors from galaxy.util.json import safe_dumps -from galaxy.util.odict import odict from galaxy.web import ( expose_api, expose_api_anonymous, @@ -201,9 +201,9 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): tool_version = kwd.get('tool_version', None) tool = self._get_tool(id, tool_version=tool_version, user=trans.user) - # Encode in this method to handle odict objects in tool representation. + # Encode in this method to handle OrderedDict objects in tool representation. def json_encodeify(obj): - if isinstance(obj, odict): + if isinstance(obj, OrderedDict): return dict(obj) elif isinstance(obj, map): return list(obj) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 8251004130d..cf8ad5d8f4c 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -4,6 +4,7 @@ API operations on User objects. import json import logging import re +from collections import OrderedDict import six from markupsafe import escape @@ -30,7 +31,6 @@ from galaxy.util import ( docstring_trim, listify ) -from galaxy.util.odict import odict from galaxy.web import ( expose_api, expose_api_anonymous @@ -689,7 +689,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, B inputs.append({'type': 'section', 'title': filter_title, 'name': filter_type, 'expanded': True, 'inputs': filter_inputs}) def _get_filter_types(self, trans): - return odict([('toolbox_tool_filters', {'title': 'Tools', 'config': trans.app.config.user_tool_filters}), + return OrderedDict([('toolbox_tool_filters', {'title': 'Tools', 'config': trans.app.config.user_tool_filters}), ('toolbox_section_filters', {'title': 'Sections', 'config': trans.app.config.user_tool_section_filters}), ('toolbox_label_filters', {'title': 'Labels', 'config': trans.app.config.user_tool_label_filters})]) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index a2dee924595..ad4f657c706 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -2,6 +2,7 @@ import imp import logging import os import time +from collections import OrderedDict from datetime import datetime, timedelta from string import punctuation as PUNCTUATION @@ -23,7 +24,6 @@ from galaxy.util import ( url_get ) from galaxy.util.hash_util import new_secure_hash -from galaxy.util.odict import odict from galaxy.web import url_for from galaxy.web.framework.helpers import grids, time_ago from galaxy.web.params import QuotaParamParser @@ -886,7 +886,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP def review_tool_migration_stages(self, trans, **kwd): message = escape(util.restore_text(kwd.get('message', ''))) status = util.restore_text(kwd.get('status', 'done')) - migration_stages_dict = odict() + migration_stages_dict = OrderedDict() # FIXME: this isn't valid in an installed context migration_scripts_dir = os.path.abspath(os.path.join(trans.app.config.root, 'lib', 'tool_shed', 'galaxy_install', 'migrate', 'versions')) modules = os.listdir(migration_scripts_dir) diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index 880354a92c2..1ecfcdb1af2 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -1,4 +1,5 @@ import logging +from collections import OrderedDict from markupsafe import escape from six import string_types @@ -15,7 +16,6 @@ from galaxy.model.item_attrs import ( UsesItemRatings ) from galaxy.util import listify, Params, parse_int, sanitize_text -from galaxy.util.odict import odict from galaxy.web import url_for from galaxy.web.framework.helpers import grids, iff, time_ago from galaxy.webapps.base.controller import ( @@ -501,7 +501,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt items = [] # First go through and group hdas by job, if there is no job they get # added directly to items - jobs = odict() + jobs = OrderedDict() for hda in history.active_datasets: if hda.visible is False: continue @@ -525,7 +525,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt else: jobs[job] = [(hda, None)] # Second, go through the jobs and connect to workflows - wf_invocations = odict() + wf_invocations = OrderedDict() for job, hdas in jobs.items(): # Job is attached to a workflow step, follow it to the # workflow_invocation and group diff --git a/lib/galaxy/webapps/reports/framework/grids.py b/lib/galaxy/webapps/reports/framework/grids.py index 677afa43847..daf17991626 100644 --- a/lib/galaxy/webapps/reports/framework/grids.py +++ b/lib/galaxy/webapps/reports/framework/grids.py @@ -1,5 +1,6 @@ import logging import math +from collections import OrderedDict from json import dumps, loads from markupsafe import escape @@ -8,7 +9,6 @@ from sqlalchemy.sql.expression import and_, false, func, null, or_, true from galaxy.model.item_attrs import get_foreign_key, UsesAnnotations, UsesItemRatings from galaxy.util import sanitize_text, unicodify -from galaxy.util.odict import odict from galaxy.web.framework import decorators, url_for from galaxy.web.framework.helpers import iff @@ -765,7 +765,7 @@ class SharingStatusColumn(GridColumn): def get_accepted_filters(self): """ Returns a list of accepted filters for this column. """ - accepted_filter_labels_and_vals = odict() + accepted_filter_labels_and_vals = OrderedDict() accepted_filter_labels_and_vals["private"] = "private" accepted_filter_labels_and_vals["shared"] = "shared" accepted_filter_labels_and_vals["accessible"] = "accessible" diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py index 760e065b1ee..aa655aaecc5 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -1,4 +1,5 @@ import logging +from collections import OrderedDict from sqlalchemy import ( and_, @@ -12,7 +13,6 @@ from galaxy import ( util, web ) -from galaxy.util.odict import odict from galaxy.web.form_builder import CheckboxField from galaxy.webapps.base.controller import BaseUIController from galaxy.webapps.tool_shed.util import ratings_util @@ -240,7 +240,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings): status = kwd.get('status', 'done') review_id = kwd.get('id', None) review = review_util.get_review(trans.app, review_id) - components_dict = odict() + components_dict = OrderedDict() for component in review_util.get_components(trans.app): components_dict[component.name] = dict(component=component, component_review=None) repository = review.repository @@ -486,7 +486,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings): repo = hg_util.get_repo_for_repository(trans.app, repository=repository) metadata_revision_hashes = [metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions] reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews] - reviews_dict = odict() + reviews_dict = OrderedDict() for changeset in hg_util.get_reversed_changelog_changesets(repo): ctx = repo.changectx(changeset) changeset_revision = str(ctx) diff --git a/lib/galaxy/workflow/extract.py b/lib/galaxy/workflow/extract.py index 53e926ec9f7..77c00e2c5f7 100644 --- a/lib/galaxy/workflow/extract.py +++ b/lib/galaxy/workflow/extract.py @@ -2,6 +2,7 @@ histories. """ import logging +from collections import OrderedDict from galaxy import exceptions, model from galaxy.tool_util.parser import ToolOutputCollectionPart @@ -14,7 +15,6 @@ from galaxy.tools.parameters.grouping import ( Repeat, Section ) -from galaxy.util.odict import odict from .steps import ( attach_ordered_steps, order_workflow_steps_with_levels @@ -199,7 +199,7 @@ class WorkflowSummary(object): history = trans.get_history() self.history = history self.warnings = set() - self.jobs = odict() + self.jobs = OrderedDict() self.job_id2representative_job = {} # map a non-fake job id to its representative job self.implicit_map_jobs = [] self.collection_types = {} diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 38b75969b87..1741d9d88e1 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -4,6 +4,7 @@ Modules used in building workflows import json import logging import re +from collections import OrderedDict from xml.etree.ElementTree import ( Element, XML @@ -50,7 +51,6 @@ from galaxy.tools.parameters.wrapped import make_dict_copy from galaxy.util import unicodify from galaxy.util.bunch import Bunch from galaxy.util.json import safe_loads -from galaxy.util.odict import odict from galaxy.util.rules_dsl import RuleSet from galaxy.util.template import fill_template from tool_shed.util import common_util @@ -681,7 +681,7 @@ class InputParameterModule(WorkflowModule): # item 0 is option description, item 1 is value, item 2 is "selected" option[2] = True input_parameter_type.static_options[i] = tuple(option) - return odict([("parameter_type", input_parameter_type), + return OrderedDict([("parameter_type", input_parameter_type), ("optional", BooleanToolParameter(None, Element("param", name="optional", label="Optional", type="boolean", value=optional)))]) def get_runtime_inputs(self, **kwds): diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py index ae20dcb943f..57bf4aaa374 100644 --- a/lib/galaxy/workflow/run.py +++ b/lib/galaxy/workflow/run.py @@ -1,9 +1,9 @@ import logging import uuid +from collections import OrderedDict from galaxy import model from galaxy.util import ExecutionTimer -from galaxy.util.odict import odict from galaxy.workflow import modules from galaxy.workflow.run_request import ( workflow_request_to_run_config, @@ -273,7 +273,7 @@ STEP_OUTPUT_DELAYED = object() class WorkflowProgress(object): def __init__(self, workflow_invocation, inputs_by_step_id, module_injector, param_map, jobs_per_scheduling_iteration=-1): - self.outputs = odict() + self.outputs = OrderedDict() self.module_injector = module_injector self.workflow_invocation = workflow_invocation self.inputs_by_step_id = inputs_by_step_id diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 46b7d1bffeb..3d12e30f26d 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -4,6 +4,7 @@ import os import shutil import tarfile import tempfile +from collections import OrderedDict from time import gmtime, strftime import requests @@ -12,7 +13,6 @@ from sqlalchemy import and_, false import tool_shed.repository_types.util as rt_util from galaxy import web from galaxy.util import asbool, build_url, CHUNK_SIZE -from galaxy.util.odict import odict from galaxy.util.path import safe_relpath from tool_shed.dependencies import attribute_handlers from tool_shed.dependencies.repository.relation_builder import RelationBuilder @@ -115,7 +115,7 @@ class ExportRepositoryManager(object): return repositories_archive, error_messages def generate_export_elem(self): - sub_elements = odict() + sub_elements = OrderedDict() sub_elements['export_time'] = strftime('%a, %d %b %Y %H:%M:%S +0000', gmtime()) sub_elements['tool_shed'] = str(self.tool_shed_url.rstrip('/')) sub_elements['repository_name'] = str(self.repository.name) @@ -261,8 +261,8 @@ class ExportRepositoryManager(object): generated attributes will be contained within the tag, while the sub_elements will be tag sets contained within the tag set. """ - attributes = odict() - sub_elements = odict() + attributes = OrderedDict() + sub_elements = OrderedDict() attributes['name'] = str(repository.name) attributes['type'] = str(repository.type) # We have to associate the public username since the user_id will be different between tool sheds. diff --git a/lib/tool_shed/dependencies/attribute_handlers.py b/lib/tool_shed/dependencies/attribute_handlers.py index 2f021c5c085..b2f8b6f4fdf 100644 --- a/lib/tool_shed/dependencies/attribute_handlers.py +++ b/lib/tool_shed/dependencies/attribute_handlers.py @@ -1,8 +1,8 @@ import copy import logging +from collections import OrderedDict from galaxy.util import asbool -from galaxy.util.odict import odict from galaxy.web import url_for from tool_shed.dependencies.tool import tag_attribute_handler from tool_shed.repository_types.util import REPOSITORY_DEPENDENCY_DEFINITION_FILENAME @@ -71,8 +71,8 @@ class RepositoryDependencyAttributeHandler(object): if len(sub_elems) > 0: # At this point, a tag will point only to a package. # - # Coerce the list to an odict(). - sub_elements = odict() + # Coerce the list to an OrderedDict(). + sub_elements = OrderedDict() packages = [] for sub_elem in sub_elems: sub_elem_type = sub_elem.tag @@ -88,7 +88,7 @@ class RepositoryDependencyAttributeHandler(object): # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes # from the tag. if toolshed or changeset_revision: - attributes = odict() + attributes = OrderedDict() attributes['name'] = name attributes['owner'] = owner prior_installation_required = elem.get('prior_installation_required') diff --git a/lib/tool_shed/galaxy_install/migrate/check.py b/lib/tool_shed/galaxy_install/migrate/check.py index be4ee9c4620..570334569db 100644 --- a/lib/tool_shed/galaxy_install/migrate/check.py +++ b/lib/tool_shed/galaxy_install/migrate/check.py @@ -2,12 +2,12 @@ import logging import os import subprocess import sys +from collections import OrderedDict from migrate.versioning import repository, schema from sqlalchemy import create_engine, MetaData, Table from galaxy.util import get_executable -from galaxy.util.odict import odict from tool_shed.util import common_util log = logging.getLogger(__name__) @@ -33,7 +33,7 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options={}): tool_shed_accessible = False if app.new_installation: # New installations will not be missing tools, so we don't need to worry about them. - missing_tool_configs_dict = odict() + missing_tool_configs_dict = OrderedDict() else: tool_panel_configs = common_util.get_non_shed_tool_panel_configs(app) if tool_panel_configs: @@ -47,7 +47,7 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options={}): # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True - missing_tool_configs_dict = odict() + missing_tool_configs_dict = OrderedDict() have_tool_dependencies = False for k, v in missing_tool_configs_dict.items(): if v: diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index e08f8227fb2..431d0ac38d5 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -8,11 +8,11 @@ import os import shutil import tempfile import threading +from collections import OrderedDict from galaxy import util from galaxy.tools.toolbox import ToolSection from galaxy.tools.toolbox.parser import ensure_tool_conf_item -from galaxy.util.odict import odict from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager @@ -112,7 +112,7 @@ class ToolMigrationManager(object): # tool_shed_accessible to True so that the value of migrate_tools.version can # be correctly set in the database. tool_shed_accessible = True - missing_tool_configs_dict = odict() + missing_tool_configs_dict = OrderedDict() if tool_shed_accessible: if len(self.proprietary_tool_confs) == 1: plural = '' @@ -386,7 +386,7 @@ class ToolMigrationManager(object): entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process. """ tool_configs_to_filter = [] - tool_panel_dict_for_display = odict() + tool_panel_dict_for_display = OrderedDict() if self.tool_path: repo_install_dir = os.path.join(self.tool_path, relative_install_dir) else: diff --git a/lib/tool_shed/repository_types/registry.py b/lib/tool_shed/repository_types/registry.py index 35563f74c8c..7a9b7b9f8bf 100644 --- a/lib/tool_shed/repository_types/registry.py +++ b/lib/tool_shed/repository_types/registry.py @@ -1,6 +1,6 @@ import logging +from collections import OrderedDict -from galaxy.util.odict import odict from . import ( repository_suite_definition, tool_dependency_definition, @@ -13,7 +13,7 @@ log = logging.getLogger(__name__) class Registry(object): def __init__(self): - self.repository_types_by_label = odict() + self.repository_types_by_label = OrderedDict() self.repository_types_by_label['unrestricted'] = unrestricted.Unrestricted() self.repository_types_by_label['repository_suite_definition'] = repository_suite_definition.RepositorySuiteDefinition() self.repository_types_by_label['tool_dependency_definition'] = tool_dependency_definition.ToolDependencyDefinition() diff --git a/lib/tool_shed/tool_shed_registry.py b/lib/tool_shed/tool_shed_registry.py index 80528095df9..4a26970a24c 100644 --- a/lib/tool_shed/tool_shed_registry.py +++ b/lib/tool_shed/tool_shed_registry.py @@ -1,9 +1,9 @@ import logging import xml.etree.ElementTree +from collections import OrderedDict from six.moves.urllib import request as urlrequest -from galaxy.util.odict import odict from tool_shed.util import common_util, xml_util log = logging.getLogger(__name__) @@ -18,8 +18,8 @@ DEFAULT_TOOL_SHEDS_CONF_XML = """ class Registry(object): def __init__(self, config=None): - self.tool_sheds = odict() - self.tool_sheds_auth = odict() + self.tool_sheds = OrderedDict() + self.tool_sheds_auth = OrderedDict() if config: # Parse tool_sheds_conf.xml tree, error_message = xml_util.parse_xml(config) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py deleted file mode 120000 index 71148b57d1c..00000000000 --- a/lib/tool_shed/util/common_util.py +++ /dev/null @@ -1 +0,0 @@ -../../galaxy/util/tool_shed/common_util.py \ No newline at end of file diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py new file mode 100644 index 00000000000..c7863da53f8 --- /dev/null +++ b/lib/tool_shed/util/common_util.py @@ -0,0 +1,362 @@ +import json +import logging +import os +from collections import OrderedDict + +from six.moves.urllib.parse import urljoin + +from galaxy import util +from galaxy.web import url_for +from tool_shed.util import encoding_util, xml_util + +log = logging.getLogger(__name__) + +REPOSITORY_OWNER = 'devteam' + + +def accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies): + if tool_shed_accessible: + if tool_dependencies: + for tool_dependency in tool_dependencies: + if tool_dependency not in all_tool_dependencies: + all_tool_dependencies.append(tool_dependency) + return all_tool_dependencies + + +def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_script_number): + # Get the 000x_tools.xml file associated with the current migrate_tools version number. + tools_xml_file_path = os.path.abspath(os.path.join('scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number)) + # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. + migrated_tool_configs_dict = OrderedDict() + tree, error_message = xml_util.parse_xml(tools_xml_file_path) + if tree is None: + return False, OrderedDict() + root = tree.getroot() + tool_shed = root.get('name') + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) + # The default behavior is that the tool shed is down. + tool_shed_accessible = False + missing_tool_configs_dict = OrderedDict() + if tool_shed_url: + for elem in root: + if elem.tag == 'repository': + repository_dependencies = [] + all_tool_dependencies = [] + repository_name = elem.get('name') + changeset_revision = elem.get('changeset_revision') + tool_shed_accessible, repository_dependencies_dict = get_repository_dependencies(app, + tool_shed_url, + repository_name, + REPOSITORY_OWNER, + changeset_revision) + if tool_shed_accessible: + # Accumulate all tool dependencies defined for repository dependencies for display to the user. + for rd_key, rd_tups in repository_dependencies_dict.items(): + if rd_key in ['root_key', 'description']: + continue + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + parse_repository_dependency_tuple(rd_tup) + tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, + tool_shed_url, + name, + owner, + changeset_revision) + all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) + tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, + tool_shed_url, + repository_name, + REPOSITORY_OWNER, + changeset_revision) + all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) + for tool_elem in elem.findall('tool'): + tool_config_file_name = tool_elem.get('file') + if tool_config_file_name: + # We currently do nothing with repository dependencies except install them (we do not display repositories that will be + # installed to the user). However, we'll store them in the following dictionary in case we choose to display them in the + # future. + dependencies_dict = dict(tool_dependencies=all_tool_dependencies, + repository_dependencies=repository_dependencies) + migrated_tool_configs_dict[tool_config_file_name] = dependencies_dict + else: + break + if tool_shed_accessible: + # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. + for tool_panel_config in tool_panel_configs: + tree, error_message = xml_util.parse_xml(tool_panel_config) + if tree: + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict) + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set(section_elem, migrated_tool_configs_dict, missing_tool_configs_dict) + else: + exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % (tool_shed, app.config.tool_sheds_config) + exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' + raise Exception(exception_msg) + return tool_shed_accessible, missing_tool_configs_dict + + +def check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict): + file_path = elem.get('file', None) + if file_path: + name = os.path.basename(file_path) + for migrated_tool_config in migrated_tool_configs_dict.keys(): + if migrated_tool_config in [file_path, name]: + missing_tool_configs_dict[name] = migrated_tool_configs_dict[migrated_tool_config] + return missing_tool_configs_dict + + +def generate_clone_url_for_installed_repository(app, repository): + """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed)) + return util.build_url(tool_shed_url, pathspec=['repos', str(repository.owner), str(repository.name)]) + + +def generate_clone_url_for_repository_in_tool_shed(user, repository): + """Generate the URL for cloning a repository that is in the tool shed.""" + base_url = url_for('/', qualified=True).rstrip('/') + if user: + protocol, base = base_url.split('://') + username = '%s@' % user.username + return '%s://%s%s/repos/%s/%s' % (protocol, username, base, repository.user.username, repository.name) + else: + return '%s/repos/%s/%s' % (base_url, repository.user.username, repository.name) + + +def generate_clone_url_from_repo_info_tup(app, repo_info_tup): + """Generate the URL for cloning a repository given a tuple of toolshed, name, owner, changeset_revision.""" + # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False] + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + parse_repository_dependency_tuple(repo_info_tup) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, toolshed) + # Don't include the changeset_revision in clone urls. + return util.build_url(tool_shed_url, pathspec=['repos', owner, name]) + + +def get_non_shed_tool_panel_configs(app): + """Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.""" + config_filenames = [] + for config_filename in app.config.tool_configs: + # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. + # + tree, error_message = xml_util.parse_xml(config_filename) + if tree is None: + continue + root = tree.getroot() + tool_path = root.get('tool_path', None) + if tool_path is None: + config_filenames.append(config_filename) + return config_filenames + + +def get_repository_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): + repository_dependencies_dict = {} + tool_shed_accessible = True + params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) + pathspec = ['repository', 'get_repository_dependencies'] + try: + raw_text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) + tool_shed_accessible = True + except Exception as e: + tool_shed_accessible = False + log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) + if tool_shed_accessible: + if len(raw_text) > 2: + encoded_text = json.loads(util.unicodify(raw_text)) + repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text) + return tool_shed_accessible, repository_dependencies_dict + + +def get_protocol_from_tool_shed_url(tool_shed_url): + """Return the protocol from the received tool_shed_url if it exists.""" + try: + if tool_shed_url.find('://') > 0: + return tool_shed_url.split('://')[0].lower() + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if tool_shed_url is not None: + log.exception("Handled exception getting the protocol from Tool Shed URL %s", str(tool_shed_url)) + # Default to HTTP protocol. + return 'http' + + +def get_tool_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): + tool_dependencies = [] + tool_shed_accessible = True + params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) + pathspec = ['repository', 'get_tool_dependencies'] + try: + text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) + tool_shed_accessible = True + except Exception as e: + tool_shed_accessible = False + log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) + if tool_shed_accessible: + if text: + tool_dependencies_dict = encoding_util.tool_shed_decode(text) + for requirements_dict in tool_dependencies_dict.values(): + tool_dependency_name = requirements_dict['name'] + tool_dependency_version = requirements_dict['version'] + tool_dependency_type = requirements_dict['type'] + tool_dependencies.append((tool_dependency_name, tool_dependency_version, tool_dependency_type)) + return tool_shed_accessible, tool_dependencies + + +def get_tool_shed_repository_ids(as_string=False, **kwd): + tsrid = kwd.get('tool_shed_repository_id', None) + tsridslist = util.listify(kwd.get('tool_shed_repository_ids', None)) + if not tsridslist: + tsridslist = util.listify(kwd.get('id', None)) + if tsridslist is not None: + if tsrid is not None and tsrid not in tsridslist: + tsridslist.append(tsrid) + if as_string: + return ','.join(tsridslist) + return tsridslist + else: + tsridslist = util.listify(kwd.get('ordered_tsr_ids', None)) + if tsridslist is not None: + if as_string: + return ','.join(tsridslist) + return tsridslist + if as_string: + return '' + return [] + + +def get_tool_shed_url_from_tool_shed_registry(app, tool_shed): + """ + The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is + something like: http://toolshed.g2.bx.psu.edu/ + """ + cleaned_tool_shed = remove_protocol_from_tool_shed_url(tool_shed) + for shed_url in app.tool_shed_registry.tool_sheds.values(): + if shed_url.find(cleaned_tool_shed) >= 0: + if shed_url.endswith('/'): + shed_url = shed_url.rstrip('/') + return shed_url + # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. + return None + + +def get_tool_shed_repository_url(app, tool_shed, owner, name): + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) + if tool_shed_url: + # Append a slash to the tool shed URL, because urlparse.urljoin will eliminate + # the last part of a URL if it does not end with a forward slash. + tool_shed_url = '%s/' % tool_shed_url + return urljoin(tool_shed_url, 'view/%s/%s' % (owner, name)) + return tool_shed_url + + +def get_user_by_username(app, username): + """Get a user from the database by username.""" + sa_session = app.model.context.current + try: + user = sa_session.query(app.model.User) \ + .filter(app.model.User.table.c.username == username) \ + .one() + return user + except Exception: + return None + + +def handle_galaxy_url(trans, **kwd): + galaxy_url = kwd.get('galaxy_url', None) + if galaxy_url: + trans.set_cookie(galaxy_url, name='toolshedgalaxyurl') + else: + galaxy_url = trans.get_cookie(name='toolshedgalaxyurl') + return galaxy_url + + +def handle_tool_shed_url_protocol(app, shed_url): + """Handle secure and insecure HTTP protocol since they may change over time.""" + try: + if app.name == 'galaxy': + url = remove_protocol_from_tool_shed_url(shed_url) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, url) + else: + tool_shed_url = str(url_for('/', qualified=True)).rstrip('/') + return tool_shed_url + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if shed_url is not None: + log.exception("Handled exception removing protocol from URL %s", str(shed_url)) + return shed_url + + +def parse_repository_dependency_tuple(repository_dependency_tuple, contains_error=False): + # Default both prior_installation_required and only_if_compiling_contained_td to False in cases where metadata should be reset on the + # repository containing the repository_dependency definition. + prior_installation_required = 'False' + only_if_compiling_contained_td = 'False' + if contains_error: + if len(repository_dependency_tuple) == 5: + tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple + elif len(repository_dependency_tuple) == 6: + tool_shed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple + elif len(repository_dependency_tuple) == 7: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ + repository_dependency_tuple + return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error + else: + if len(repository_dependency_tuple) == 4: + tool_shed, name, owner, changeset_revision = repository_dependency_tuple + elif len(repository_dependency_tuple) == 5: + tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple + elif len(repository_dependency_tuple) == 6: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tuple + return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td + + +def remove_port_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the port if it exists.""" + try: + if tool_shed_url.find(':') > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + new_tool_shed_url = tool_shed_url.split(':')[0] + else: + new_tool_shed_url = tool_shed_url + return new_tool_shed_url.rstrip('/') + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if tool_shed_url is not None: + log.exception("Handled exception removing the port from Tool Shed URL %s", str(tool_shed_url)) + return tool_shed_url + + +def remove_protocol_and_port_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the protocol and/or port if either exists.""" + tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url) + tool_shed = remove_port_from_tool_shed_url(tool_shed) + return tool_shed + + +def remove_protocol_and_user_from_clone_url(repository_clone_url): + """Return a URL that can be used to clone a repository, eliminating the protocol and user if either exists.""" + if repository_clone_url.find('@') > 0: + # We have an url that includes an authenticated user, something like: + # http://test@bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split('@') + tmp_url = items[1] + elif repository_clone_url.find('//') > 0: + # We have an url that includes only a protocol, something like: + # http://bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split('//') + tmp_url = items[1] + else: + tmp_url = repository_clone_url + return tmp_url.rstrip('/') + + +def remove_protocol_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the protocol if it exists.""" + return util.remove_protocol_from_url(tool_shed_url) diff --git a/lib/tool_shed/util/review_util.py b/lib/tool_shed/util/review_util.py index 7e87d6f6e34..61fb714e4e6 100644 --- a/lib/tool_shed/util/review_util.py +++ b/lib/tool_shed/util/review_util.py @@ -1,8 +1,8 @@ import logging +from collections import OrderedDict from sqlalchemy import and_ -from galaxy.util.odict import odict from tool_shed.util import hg_util log = logging.getLogger(__name__) @@ -74,7 +74,7 @@ def get_previous_repository_reviews(app, repository, changeset_revision): """ repo = hg_util.get_repo_for_repository(app, repository=repository) reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews] - previous_reviews_dict = odict() + previous_reviews_dict = OrderedDict() for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision): previous_changeset_revision = str(repo.changectx(changeset)) if previous_changeset_revision in reviewed_revision_hashes: diff --git a/lib/tool_shed/utility_containers/utility_container_manager.py b/lib/tool_shed/utility_containers/utility_container_manager.py index 6d0e098c224..5ed2aa09c8c 100644 --- a/lib/tool_shed/utility_containers/utility_container_manager.py +++ b/lib/tool_shed/utility_containers/utility_container_manager.py @@ -850,7 +850,7 @@ class UtilityContainerManager(object): def prune_repository_dependencies(self, folder): """ Since the object used to generate a repository dependencies container is a dictionary - and not an odict() (it must be json-serialize-able), the order in which the dictionary + and not an OrderedDict() (it must be json-serialize-able), the order in which the dictionary is processed to create the container sometimes results in repository dependency entries in a folder that also includes the repository dependency as a sub-folder (if the repository dependency has its own repository dependency). This method will remove all repository diff --git a/test/unit/tools/test_execution.py b/test/unit/tools/test_execution.py index ed745a7502f..8a84f58d8df 100644 --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -1,5 +1,6 @@ """ Test Tool execution and state handling logic. """ +from collections import OrderedDict from unittest import TestCase import webob.exc @@ -7,7 +8,6 @@ import webob.exc import galaxy.model from galaxy.tools.parameters import params_to_incoming from galaxy.util.bunch import Bunch -from galaxy.util.odict import odict from .. import tools_support BASE_REPEAT_TOOL_CONTENTS = ''' @@ -185,7 +185,7 @@ class MockAction(object): if num_calls > self.error_message_after_excution: return None, "Test Error Message" - return galaxy.model.Job(), odict(dict(out1="1")) + return galaxy.model.Job(), OrderedDict(dict(out1="1")) def raise_exception(self, after_execution=0): self.exception_after_exection = after_execution diff --git a/tools/filters/secure_hash_message_digest.py b/tools/filters/secure_hash_message_digest.py index f3b6ba7a2a1..8bd7c596542 100644 --- a/tools/filters/secure_hash_message_digest.py +++ b/tools/filters/secure_hash_message_digest.py @@ -5,8 +5,7 @@ A script for calculating secure hashes / message digests. """ import hashlib import optparse - -from galaxy.util.odict import odict +from collections import OrderedDict HASH_ALGORITHMS = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'] CHUNK_SIZE = 2 ** 20 # 1mb @@ -20,7 +19,7 @@ def __main__(): parser.add_option('-o', '--output', dest='output', action='store', type="string", help='Output filename') (options, args) = parser.parse_args() - algorithms = odict() + algorithms = OrderedDict() for algorithm in options.algorithms: assert algorithm in HASH_ALGORITHMS, "Invalid algorithm specified: %s" % (algorithm) assert algorithm not in algorithms, "Specify each algorithm only once."