opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 13:50:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release_21.01' into dev

Browse Source
This commit is contained in:
mvdbeek
2021-02-08 16:01:53 +01:00
parent f6623d3a14 5078e34de9
commit dd7aa6ec38
91 changed files with 511 additions and 874 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/minikube-test-setup/deployment.yaml
+32
View File
@@ -0,0 +1,32 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: testing
name: testing
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: test
template:
metadata:
labels:
app.kubernetes.io/name: test
spec:
containers:
- image: postgres:12
name: postgres
ports:
- containerPort: 5432
env:
- name: POSTGRES_DB
value: postgres
- name: POSTGRES_USER
value: postgres
- name: POSTGRES_PASSWORD
value: postgres
- image: rabbitmq
name: rabbitmq
ports:
- containerPort: 5672
.ci/minikube-test-setup/start_services.sh
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -ex
SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
kubectl apply -f "$SCRIPTDIR/deployment.yaml"
kubectl expose deployment testing --type=LoadBalancer --name=testing-service
CLUSTER_IP=$(kubectl get service testing-service -o jsonpath='{.spec.clusterIP}')
GALAXY_TEST_DBURI="postgresql://postgres:postgres@${CLUSTER_IP}:5432/galaxy?client_encoding=utf-8"
GALAXY_TEST_AMQP_URL="amqp://${CLUSTER_IP}:5672)//"
export GALAXY_TEST_DBURI
export GALAXY_TEST_AMQP_URL
.github/workflows/integration.yaml
+9 -2
View File
@@ -6,7 +6,7 @@ env:
jobs:
test:
name: Test
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
@@ -14,7 +14,7 @@ jobs:
subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)']
services:
postgres:
image: postgres:11
image: postgres:13
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
@@ -60,8 +60,15 @@ jobs:
run: sudo apt-get update && sudo apt-get install ffmpeg -y
if: matrix.subset == 'upload_datatype'
- name: Run tests
if: matrix.subset != 'kubernetes'
run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"'
working-directory: 'galaxy root'
- name: Run tests
if: matrix.subset == 'kubernetes'
run: |
. .ci/minikube-test-setup/start_services.sh
./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"
working-directory: 'galaxy root'
- uses: actions/upload-artifact@v2
if: failure()
with:
client/src/components/Common/SlugInput.vue
+1
View File
@@ -18,6 +18,7 @@ export default {
props: {
slug: {
type: String,
required: true,
},
},
data() {
client/src/components/Sharing.vue
+36 -40
View File
@@ -12,7 +12,7 @@
</div>
<div v-else>
<b-form-checkbox switch class="make-accessible" v-model="item.importable" @change="onImportable">
Make {{ model_class }} accessible.
Make {{ model_class }} accessible
</b-form-checkbox>
<b-form-checkbox
v-if="item.importable"
@@ -22,38 +22,38 @@
@change="onPublish"
>
Make {{ model_class }} publicly available in
<a :href="published_url" target="_top">Published {{ plural_name }}</a> section.
<a :href="published_url" target="_top">Published {{ plural_name }}</a>
</b-form-checkbox>
<br />
<div>
<div v-if="item.importable">
<div>
This {{ model_class }} is currently <strong>{{ itemStatus }}</strong
>.
</div>
<p>Anyone can view and import this {{ model_class }} by visiting the following URL:</p>
<blockquote>
<b-button title="Edit URL" @click="onEdit" v-b-tooltip.hover variant="link" size="sm">
<font-awesome-icon icon="edit" />
</b-button>
<b-button id="tooltip-clipboard" @click="onCopy" @mouseout="onCopyOut" variant="link" size="sm">
<font-awesome-icon icon="link" />
</b-button>
<b-tooltip target="tooltip-clipboard" triggers="hover">
{{ tooltipClipboard }}
</b-tooltip>
<a v-if="showUrl" id="item-url" :href="itemUrl" target="_top" class="ml-2">
{{ itemUrl }}
</a>
<span v-else id="item-url-text">
{{ itemUrlParts[0] }}<SlugInput class="ml-1" :slug="itemUrlParts[1]" @onChange="onChange" />
</span>
</blockquote>
</div>
<div v-else>
Access to this {{ model_class }} is currently restricted so that only you and the users listed below
can access it. Note that sharing a History will also allow access to all of its datasets.
<div v-if="item.importable">
<div>
This {{ model_class }} is currently <strong>{{ itemStatus }}</strong
>.
</div>
<p>Anyone can view and import this {{ model_class }} by visiting the following URL:</p>
<blockquote>
<b-button title="Edit URL" @click="onEdit" v-b-tooltip.hover variant="link" size="sm">
<font-awesome-icon icon="edit" />
</b-button>
<b-button id="tooltip-clipboard" @click="onCopy" @mouseout="onCopyOut" variant="link" size="sm">
<font-awesome-icon icon="link" />
</b-button>
<b-tooltip target="tooltip-clipboard" triggers="hover">
{{ tooltipClipboard }}
</b-tooltip>
<a v-if="showUrl" id="item-url" :href="itemUrl" target="_top" class="ml-2">
url:
{{ itemUrl }}
</a>
<span v-else id="item-url-text">
slug:
{{ itemUrlParts[0] }}<SlugInput class="ml-1" :slug="itemUrlParts[1]" @onChange="onChange" />
</span>
</blockquote>
</div>
<div v-else>
Access to this {{ model_class }} is currently restricted so that only you and the users listed below can
access it. Note that sharing a History will also allow access to all of its datasets.
</div>
<br />
<h4>Share {{ model_class }} with Individual Users</h4>
@@ -127,7 +127,7 @@ export default {
errMsg: null,
item: {
title: "title",
username_and_slug: "username_and_slug",
username_and_slug: "username/slug",
importable: false,
published: false,
users_shared_with: [],
@@ -205,16 +205,10 @@ export default {
},
onImportable(importable) {
if (importable) {
this.setSharing("make_accessible_via_link");
if (this.item.published) {
this.setSharing("publish");
} else {
this.setSharing("unpublish");
}
this.setSharing(`make_accessible_via_link-${this.item.published ? "publish" : "unpublish"}`);
} else {
this.item.published = false;
this.setSharing("disable_link_access");
this.setSharing("unpublish");
this.setSharing("disable_link_access-unpublish");
}
},
onPublish(published) {
@@ -253,12 +247,14 @@ export default {
action: action,
user_id: user_id,
};
axios
return axios
.post(`${getAppRoot()}api/${this.pluralNameLower}/${this.id}/sharing`, data)
.then((response) => {
if (response.data.skipped) {
this.errMsg = "Some of the items within this object were not published due to an error.";
}
this.item = response.data;
this.ready = true;
})
.catch((error) => (this.errMsg = error.response.data.err_msg));
},
client/src/onload/standardInit.js
+2 -2
View File
@@ -44,8 +44,8 @@ export function standardInit(label = "Galaxy", appFactory = defaultAppFactory) {
// functions even if they are registered super-late because combineLatest
// will not remake a the existing Galaxy or config objects, it'll just run
// the new batch of freshly registered init functions
combineLatest(config$, galaxy$, initializations$).subscribe(([config, galaxy, inits]) => {
console.groupCollapsed(`runInitializations`, label, serverPath());
combineLatest([config$, galaxy$, initializations$]).subscribe(([config, galaxy, inits]) => {
console.group(`runInitializations`, label, serverPath());
inits.forEach((fn) => fn(galaxy, config));
clearInitQueue();
console.groupEnd();
config/plugins/webhooks/gtn/script.js
+66 -2
View File
@@ -1,4 +1,5 @@
var gtnWebhookLoaded = false;
var lastUpdate = 0;
function removeOverlay() {
document.getElementById("gtn-container").style.visibility = "hidden";
@@ -8,9 +9,43 @@ function showOverlay() {
document.getElementById("gtn-container").style.visibility = "visible";
}
function getIframeUrl() {
var loc;
try {
loc = document.getElementById("gtn-embed").contentWindow.location.pathname;
} catch (e) {
loc = null;
}
return loc;
}
function getIframeScroll() {
var loc;
try {
loc = parseInt(document.getElementById("gtn-embed").contentWindow.scrollY);
} catch (e) {
loc = 0;
}
return loc;
}
function restoreLocation() {}
function persistLocation() {
// Don't save every scroll event.
var time = new Date().getTime();
if (time - lastUpdate < 1000) {
return;
}
lastUpdate = time;
window.localStorage.setItem("gtn-in-galaxy", `${getIframeScroll()} ${getIframeUrl()}`);
}
function addIframe() {
let url, message;
let url, message, onloadscroll;
gtnWebhookLoaded = true;
let storedData = false;
let safe = false;
// Test for the presence of /training-material/. If that is available we
// can opt in the fancy click-to-run features. Otherwise we fallback to
@@ -24,7 +59,19 @@ function addIframe() {
<a href="https://docs.galaxyproject.org/en/master/admin/special_topics/gtn.html">Click to run</a> unavailable.
</span>`;
} else {
url = "/training-material/";
safe = true;
var storedLocation = window.localStorage.getItem("gtn-in-galaxy");
if (
storedLocation !== null &&
storedLocation.split(" ")[1] !== undefined &&
storedLocation.split(" ")[1].startsWith("/training-material/")
) {
onloadscroll = storedLocation.split(" ")[0];
url = storedLocation.split(" ")[1];
} else {
url = "/training-material/";
}
message = "";
}
})
@@ -48,8 +95,25 @@ function addIframe() {
removeOverlay();
});
// Only setup the listener if it won't crash things.
if (safe) {
// Listen to the scroll position
document.getElementById("gtn-embed").contentWindow.addEventListener("scroll", () => {
persistLocation();
});
}
// Depends on the iframe being present
document.getElementById("gtn-embed").addEventListener("load", () => {
// Save our current location when possible
if (onloadscroll !== undefined) {
document.getElementById("gtn-embed").contentWindow.scrollTo(0, parseInt(onloadscroll));
onloadscroll = undefined;
}
if (safe) {
persistLocation();
}
var gtn_tools = $("#gtn-embed").contents().find("span[data-tool]");
// Buttonify
gtn_tools.addClass("galaxy-proxy-active");
doc/source/admin/galaxy_options.rst
+7 -7
View File
@@ -3851,13 +3851,13 @@
~~~~~~~~~~~~~~~~~~~~~
:Description:
Determines how metadata will be set. Valid values are `directory`,
`extended` and `legacy`. In extended mode jobs will decide if a
tool run failed, the object stores configuration is serialized and
made available to the job and is used for writing output datasets
to the object store as part of the job and dynamic output
discovery (e.g. discovered datasets <discover_datasets>,
unpopulated collections, etc) happens as part of the job.
Determines how metadata will be set. Valid values are `directory`
and `extended`. In extended mode jobs will decide if a tool run
failed, the object stores configuration is serialized and made
available to the job and is used for writing output datasets to
the object store as part of the job and dynamic output discovery
(e.g. discovered datasets <discover_datasets>, unpopulated
collections, etc) happens as part of the job.
:Default: ``directory``
:Type: str
lib/galaxy/config/config_manage.py
+8 -11
View File
@@ -4,9 +4,6 @@ import shutil
import string
import sys
import tempfile
from collections import (
OrderedDict
)
from io import StringIO
from textwrap import TextWrapper
from typing import Any, List, NamedTuple
@@ -52,7 +49,7 @@ YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ",
RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent=" ", subsequent_indent=" ", break_long_words=False, break_on_hyphens=False)
UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml"
UWSGI_OPTIONS = OrderedDict([
UWSGI_OPTIONS = dict([
('http', {
'desc': """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use ':$default_port' to listen on all available network interfaces.""",
'default': '127.0.0.1:$default_port',
@@ -441,7 +438,7 @@ def _build_uwsgi_schema(args, app_desc):
last_line = None
current_opt = None
options = OrderedDict({})
options = {}
option = None
for line in rst_options.splitlines():
line = line.strip()
@@ -506,9 +503,9 @@ def _find_app_options(app_desc, path):
def _find_app_options_from_config_parser(p):
if not p.has_section("app:main"):
_warn(NO_APP_MAIN_MESSAGE)
app_items = OrderedDict()
app_items = {}
else:
app_items = OrderedDict(p.items("app:main"))
app_items = dict(p.items("app:main"))
return app_items
@@ -604,9 +601,9 @@ def _run_conversion(args, app_desc):
if not server_section:
_warn("No server section found, using default uwsgi server definition.")
server_config = OrderedDict()
server_config = {}
else:
server_config = OrderedDict(p.items(server_section))
server_config = dict(p.items(server_section))
app_items = _find_app_options_from_config_parser(p)
applied_filters = []
@@ -621,7 +618,7 @@ def _run_conversion(args, app_desc):
uwsgi_dict = _server_paste_to_uwsgi(app_desc, server_config, applied_filters)
app_dict = OrderedDict({})
app_dict = {}
schema = app_desc.schema
for key, value in app_items.items():
if key in ["__file__", "here"]:
@@ -778,7 +775,7 @@ def _parse_option_value(option_value):
def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
uwsgi_dict = OrderedDict()
uwsgi_dict = {}
port = server_config.get("port", app_desc.default_port)
host = server_config.get("host", "127.0.0.1")
lib/galaxy/config/sample/galaxy.yml.sample
+3 -3
View File
@@ -1902,9 +1902,9 @@ galaxy:
# database.
#enable_job_recovery: true
# Determines how metadata will be set. Valid values are `directory`,
# `extended` and `legacy`. In extended mode jobs will decide if a tool
# run failed, the object stores configuration is serialized and made
# Determines how metadata will be set. Valid values are `directory`
# and `extended`. In extended mode jobs will decide if a tool run
# failed, the object stores configuration is serialized and made
# available to the job and is used for writing output datasets to the
# object store as part of the job and dynamic output discovery (e.g.
# discovered datasets <discover_datasets>, unpopulated collections,
lib/galaxy/datatypes/binary.py
+1 -2
View File
@@ -12,7 +12,6 @@ import sys
import tarfile
import tempfile
import zipfile
from collections import OrderedDict
from json import dumps
from typing import Optional
@@ -320,7 +319,7 @@ class BamNative(CompressedArchive):
# TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
dataset.metadata.reference_names = list(bam_file.references)
dataset.metadata.reference_lengths = list(bam_file.lengths)
dataset.metadata.bam_header = OrderedDict((k, v) for k, v in bam_file.header.items())
dataset.metadata.bam_header = dict(bam_file.header.items())
dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
dataset.metadata.sort_order = dataset.metadata.bam_header.get('HD', {}).get('SO', None)
dataset.metadata.bam_version = dataset.metadata.bam_header.get('HD', {}).get('VN', None)
lib/galaxy/datatypes/data.py
+5 -6
View File
@@ -5,7 +5,6 @@ import os
import shutil
import string
import tempfile
from collections import OrderedDict
from inspect import isclass
from typing import Any, Dict, Optional
@@ -125,7 +124,7 @@ class Data(metaclass=DataMeta):
is_binary = True
# Composite datatypes
composite_type: Optional[str] = None
composite_files: Dict[str, Any] = OrderedDict()
composite_files: Dict[str, Any] = {}
primary_file_name = 'index'
# Allow user to change between this datatype and others. If left to None,
# datatype change is allowed if the datatype is not composite.
@@ -144,7 +143,7 @@ class Data(metaclass=DataMeta):
object.__init__(self, **kwd)
self.supported_display_apps = self.supported_display_apps.copy()
self.composite_files = self.composite_files.copy()
self.display_applications = OrderedDict()
self.display_applications = {}
@classmethod
def is_datatype_change_allowed(cls):
@@ -565,7 +564,7 @@ class Data(metaclass=DataMeta):
return self.display_applications.get(key, default)
def get_display_applications_by_dataset(self, dataset, trans):
rval = OrderedDict()
rval = {}
for key, value in self.display_applications.items():
value = value.filter_by_dataset(dataset, trans)
if value.links:
@@ -685,7 +684,7 @@ class Data(metaclass=DataMeta):
@property
def writable_files(self):
files = OrderedDict()
files = {}
if self.composite_type != 'auto_primary_file':
files[self.primary_file_name] = self.__new_composite_file(self.primary_file_name)
for key, value in self.get_composite_files().items():
@@ -701,7 +700,7 @@ class Data(metaclass=DataMeta):
meta_value = self.metadata_spec[composite_file.substitute_name_with_metadata].default
return key % meta_value
return key
files = OrderedDict()
files = {}
for key, value in self.composite_files.items():
files[substitute_composite_key(key, value)] = value
return files
lib/galaxy/datatypes/display_applications/application.py
+4 -5
View File
@@ -1,6 +1,5 @@
# Contains objects for using external display applications
import logging
from collections import OrderedDict
from copy import deepcopy
from urllib.parse import quote_plus
@@ -46,7 +45,7 @@ class DisplayApplicationLink:
def __init__(self, display_application):
self.display_application = display_application
self.parameters = OrderedDict() # parameters are populated in order, allowing lower listed ones to have values of higher listed ones
self.parameters = {}
self.url_param_name_map = {}
self.url = None
self.id = None
@@ -64,9 +63,9 @@ class DisplayApplicationLink:
def get_inital_values(self, data, trans):
if self.other_values:
rval = OrderedDict(self.other_values)
rval = dict(self.other_values)
else:
rval = OrderedDict()
rval = {}
rval.update({'BASE_URL': trans.request.base, 'APP': trans.app}) # trans automatically appears as a response, need to add properties of trans that we want here
BASE_PARAMS = {'qp': quote_plus_string, 'url_for': trans.app.url_for}
for key, value in BASE_PARAMS.items(): # add helper functions/variables
@@ -289,7 +288,7 @@ class DisplayApplication:
if version is None:
version = "1.0.0"
self.version = version
self.links = OrderedDict()
self.links = {}
self._filename = filename
self._elem = elem
self._data_table_versions = {}
lib/galaxy/datatypes/registry.py
+4 -5
View File
@@ -5,7 +5,6 @@ Provides mapping between extensions and datatypes, mime-types, etc.
import imp
import logging
import os
from collections import OrderedDict
from string import Template
import yaml
@@ -41,7 +40,7 @@ class Registry:
self.config = config
self.datatypes_by_extension = {}
self.mimetypes_by_extension = {}
self.datatype_converters = OrderedDict()
self.datatype_converters = {}
# Converters defined in local datatypes_conf.xml
self.converters = []
self.converter_tools = set()
@@ -58,7 +57,7 @@ class Registry:
# tool shed repositories that contain display applications.
self.proprietary_display_app_containers = []
# Map a display application id to a display application
self.display_applications = OrderedDict()
self.display_applications = {}
# The following 2 attributes are used in the to_xml_file()
# method to persist the current state into an xml file.
self.display_path_attr = None
@@ -638,7 +637,7 @@ class Registry:
else:
toolbox.register_tool(converter)
if source_datatype not in self.datatype_converters:
self.datatype_converters[source_datatype] = OrderedDict()
self.datatype_converters[source_datatype] = {}
self.datatype_converters[source_datatype][target_datatype] = converter
if not hasattr(toolbox.app, 'tool_cache') or converter.id in toolbox.app.tool_cache._new_tool_ids:
self.log.debug("Loaded converter: %s", converter.id)
@@ -863,7 +862,7 @@ class Registry:
def get_converters_by_datatype(self, ext):
"""Returns available converters by source type"""
if ext not in self._converters_by_datatype:
converters = OrderedDict()
converters = {}
source_datatype = type(self.get_datatype_by_extension(ext))
for ext2, converters_dict in self.datatype_converters.items():
converter_datatype = type(self.get_datatype_by_extension(ext2))
lib/galaxy/datatypes/util/gff_util.py
+1 -2
View File
@@ -2,7 +2,6 @@
Provides utilities for working with GFF files.
"""
import copy
from collections import OrderedDict
from bx.intervals.io import GenomicInterval, GenomicIntervalReader, MissingFieldError, NiceReaderWrapper, ParseError
from bx.tabular.io import Comment, Header
@@ -428,7 +427,7 @@ def read_unordered_gtf(iterator, strict=False):
return fields[0] + '_' + get_transcript_id(fields)
# Aggregate intervals by transcript_id and collect comments.
feature_intervals = OrderedDict()
feature_intervals = {}
comments = []
for line in iterator:
if line.startswith('#'):
lib/galaxy/dependencies/pinned-requirements.txt
+1
View File
@@ -161,6 +161,7 @@ six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" and pytho
social-auth-core==3.3.0
sortedcontainers==2.3.0
sqlalchemy-migrate==0.13.0
sqlalchemy-mutable==0.0.11
sqlalchemy-utils==0.36.7
sqlalchemy==1.3.22; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
sqlitedict==1.7.0
lib/galaxy/job_execution/output_collect.py
+3 -4
View File
@@ -4,7 +4,6 @@ import logging
import operator
import os
import re
from collections import OrderedDict
from tempfile import NamedTemporaryFile
import galaxy.model
@@ -173,7 +172,7 @@ class BaseJobContext:
pass
def find_files(self, output_name, collection, dataset_collectors):
filenames = OrderedDict()
filenames = {}
for discovered_file in discover_files(output_name, self.tool_provided_metadata, dataset_collectors, self.job_working_directory, collection):
filenames[discovered_file.path] = discovered_file
return filenames
@@ -373,7 +372,7 @@ def collect_primary_datasets(job_context, output, input_ext):
output_def = job_context.output_def(name)
if output_def is not None:
dataset_collectors = [dataset_collector(description) for description in output_def.dataset_collector_descriptions]
filenames = OrderedDict()
filenames = {}
for discovered_file in discover_files(name, job_context.tool_provided_metadata, dataset_collectors, job_working_directory, outdata):
filenames[discovered_file.path] = discovered_file
for filename_index, (filename, discovered_file) in enumerate(filenames.items()):
@@ -400,7 +399,7 @@ def collect_primary_datasets(job_context, output, input_ext):
primary_output_assigned = True
continue
if name not in primary_datasets:
primary_datasets[name] = OrderedDict()
primary_datasets[name] = {}
visible = fields_match.visible
# Create new primary dataset
new_primary_name = fields_match.name or f"{outdata.name} ({designation})"
lib/galaxy/job_execution/setup.py
+3
View File
@@ -3,6 +3,9 @@ import os
from galaxy.util import safe_makedirs
TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename']
def ensure_configs_directory(work_dir):
configs_dir = os.path.join(work_dir, "configs")
lib/galaxy/jobs/__init__.py
+4 -7
View File
@@ -40,9 +40,12 @@ from galaxy.job_execution.datasets import (
TaskPathRewriter
)
from galaxy.job_execution.output_collect import collect_extra_files
from galaxy.job_execution.setup import (
from galaxy.job_execution.setup import ( # noqa: F401
create_working_directory_for_job,
ensure_configs_directory,
# This is read by certain misbehaving tool wrappers that import Galaxy internals
TOOL_PROVIDED_JOB_METADATA_FILE,
TOOL_PROVIDED_JOB_METADATA_KEYS,
)
from galaxy.jobs.actions.post import ActionBox
from galaxy.jobs.mapper import (
@@ -72,12 +75,6 @@ from galaxy.web_stack.handlers import ConfiguresHandlers
log = logging.getLogger(__name__)
# Legacy definition - this is read by certain misbehaving tool wrappers
# that import Galaxy internals - but it shouldn't be used in Galaxy's code
# itself.
TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename']
# Override with config.default_job_shell.
DEFAULT_JOB_SHELL = '/bin/bash'
DEFAULT_LOCAL_WORKERS = 4
lib/galaxy/jobs/runners/pulsar.py
+2
View File
@@ -593,6 +593,8 @@ class PulsarJobRunner(AsynchronousJobRunner):
files_endpoint=files_endpoint,
env=env
)
# Turn MutableDict into standard dict for pulsar consumption
job_destination_params = dict(job_destination_params.items())
return self.client_manager.get_client(job_destination_params, **get_client_kwds)
def finish_job(self, job_state):
lib/galaxy/managers/collections.py
+5 -6
View File
@@ -1,5 +1,4 @@
import logging
from collections import OrderedDict
from sqlalchemy.orm import joinedload, Query
@@ -378,7 +377,7 @@ class DatasetCollectionManager:
if elements is self.ELEMENTS_UNINITIALIZED:
return
new_elements = OrderedDict()
new_elements = {}
for key, element in elements.items():
if isinstance(element, model.DatasetCollection):
continue
@@ -387,7 +386,7 @@ class DatasetCollectionManager:
continue
# element is a dict with src new_collection and
# and OrderedDict of named elements
# and dict of named elements
collection_type = element.get("collection_type")
sub_elements = element["elements"]
collection = self.create_dataset_collection(
@@ -402,7 +401,7 @@ class DatasetCollectionManager:
elements.update(new_elements)
def __load_elements(self, trans, element_identifiers, hide_source_items=False, copy_elements=False, history=None):
elements = OrderedDict()
elements = {}
for element_identifier in element_identifiers:
elements[element_identifier["name"]] = self.__load_element(trans,
element_identifier=element_identifier,
@@ -500,7 +499,7 @@ class DatasetCollectionManager:
def _build_elements_from_rule_data(self, collection_type_description, rule_set, data, sources, handle_dataset):
identifier_columns = rule_set.identifier_columns
mapping_as_dict = rule_set.mapping_as_dict
elements = OrderedDict()
elements = {}
for data_index, row_data in enumerate(data):
# For each row, find place in depth for this element.
collection_type_at_depth = collection_type_description
@@ -546,7 +545,7 @@ class DatasetCollectionManager:
sub_collection = {}
sub_collection["src"] = "new_collection"
sub_collection["collection_type"] = collection_type_at_depth.collection_type
sub_collection["elements"] = OrderedDict()
sub_collection["elements"] = {}
elements_at_depth[identifier] = sub_collection
elements_at_depth = sub_collection["elements"]
lib/galaxy/managers/executables.py
+2 -2
View File
@@ -1,6 +1,6 @@
"""Utilities for loading tools and workflows from paths for admin user requests."""
from gxformat2.converter import ordered_load
import yaml
from galaxy import exceptions
@@ -13,7 +13,7 @@ def artifact_class(trans, as_dict):
workflow_path = as_dict.get("path")
with open(workflow_path) as f:
as_dict = ordered_load(f)
as_dict = yaml.safe_load(f)
artifact_class = as_dict.get("class", None)
if artifact_class is None and "$graph" in as_dict:
lib/galaxy/managers/markdown_util.py
+2 -3
View File
@@ -18,7 +18,6 @@ import os
import re
import shutil
import tempfile
from collections import OrderedDict
import markdown
import pkg_resources
@@ -448,11 +447,11 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler):
def handle_job_metrics(self, line, job):
job_metrics = summarize_job_metrics(self.trans, job)
metrics_by_plugin = OrderedDict()
metrics_by_plugin = {}
for job_metric in job_metrics:
plugin = job_metric["plugin"]
if plugin not in metrics_by_plugin:
metrics_by_plugin[plugin] = OrderedDict()
metrics_by_plugin[plugin] = {}
metrics_by_plugin[plugin][job_metric["title"]] = job_metric["value"]
markdown = ""
for metric_plugin, metrics_for_plugin in metrics_by_plugin.items():
lib/galaxy/metadata/__init__.py
+17 -223
View File
@@ -3,17 +3,14 @@
import abc
import json
import os
import pickle
import shutil
import tempfile
from logging import getLogger
from os.path import abspath
import galaxy.model
from galaxy.model import store
from galaxy.model.metadata import FileParameter, MetadataTempFile
from galaxy.model.store import DirectoryModelExportStore
from galaxy.util import in_directory, safe_makedirs
from galaxy.util import safe_makedirs
log = getLogger(__name__)
@@ -23,7 +20,7 @@ SET_METADATA_SCRIPT = 'from galaxy_ext.metadata.set_metadata import set_metadata
def get_metadata_compute_strategy(config, job_id, metadata_strategy_override=None, tool_id=None):
metadata_strategy = metadata_strategy_override or config.metadata_strategy
if metadata_strategy == "legacy":
return JobExternalOutputMetadataWrapper(job_id)
raise Exception('legacy metadata_strategy has been removed')
elif metadata_strategy == "extended" and tool_id != "__SET_METADATA__":
return ExtendedDirectoryMetadataGenerator(job_id)
else:
@@ -130,23 +127,22 @@ class PortableDirectoryMetadataGenerator(MetadataCollectionStrategy):
outputs = {}
output_collections = {}
real_metadata_object = self.write_object_store_conf
for name, dataset in datasets_dict.items():
assert name is not None
assert name not in outputs
key = name
def _metadata_path(what):
return os.path.join(metadata_dir, f"metadata_{what}_{key}")
_initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds, real_metadata_object=real_metadata_object)
_initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds, real_metadata_object=self.write_object_store_conf)
outputs[name] = {
"filename_override": _get_filename_override(output_fnames, dataset.file_name),
"validate": validate_outputs,
"object_store_store_by": dataset.dataset.store_by,
'id': dataset.id,
'model_class': 'LibraryDatasetDatasetAssociation' if isinstance(dataset, galaxy.model.LibraryDatasetDatasetAssociation) else 'HistoryDatasetAssociation'
}
metadata_params_path = os.path.join(metadata_dir, "params.json")
@@ -159,19 +155,19 @@ class PortableDirectoryMetadataGenerator(MetadataCollectionStrategy):
"outputs": outputs,
}
# export model objects and object store configuration for extended metadata also.
export_directory = os.path.join(metadata_dir, "outputs_new")
with DirectoryModelExportStore(export_directory, for_edit=True, serialize_dataset_objects=True) as export_store:
for dataset in datasets_dict.values():
export_store.add_dataset(dataset)
for name, dataset_collection in out_collections.items():
export_store.add_dataset_collection(dataset_collection)
output_collections[name] = {
'id': dataset_collection.id,
}
if self.write_object_store_conf:
# export model objects and object store configuration for extended metadata also.
export_directory = os.path.join(metadata_dir, "outputs_new")
with DirectoryModelExportStore(export_directory, for_edit=True, serialize_dataset_objects=True) as export_store:
for dataset in datasets_dict.values():
export_store.add_dataset(dataset)
for name, dataset_collection in out_collections.items():
export_store.add_dataset_collection(dataset_collection)
output_collections[name] = {
'id': dataset_collection.id,
}
with open(os.path.join(metadata_dir, "object_store_conf.json"), "w") as f:
json.dump(object_store_conf, f)
@@ -241,195 +237,12 @@ class ExtendedDirectoryMetadataGenerator(PortableDirectoryMetadataGenerator):
return dataset
class JobExternalOutputMetadataWrapper(MetadataCollectionStrategy):
"""
Class with methods allowing set_meta() to be called externally to the
Galaxy head.
This class allows access to external metadata filenames for all outputs
associated with a job.
We will use JSON as the medium of exchange of information, except for the
DatasetInstance object which will use pickle (in the future this could be
JSONified as well)
"""
portable = False
def __init__(self, job_id):
self.job_id = job_id
def _get_output_filenames_by_dataset(self, dataset, sa_session):
if isinstance(dataset, galaxy.model.HistoryDatasetAssociation):
return sa_session.query(galaxy.model.JobExternalOutputMetadata) \
.filter_by(job_id=self.job_id,
history_dataset_association_id=dataset.id,
is_valid=True) \
.first() # there should only be one or None
elif isinstance(dataset, galaxy.model.LibraryDatasetDatasetAssociation):
return sa_session.query(galaxy.model.JobExternalOutputMetadata) \
.filter_by(job_id=self.job_id,
library_dataset_dataset_association_id=dataset.id,
is_valid=True) \
.first() # there should only be one or None
return None
def _get_dataset_metadata_key(self, dataset):
# Set meta can be called on library items and history items,
# need to make different keys for them, since ids can overlap
return "%s_%d" % (dataset.__class__.__name__, dataset.id)
def invalidate_external_metadata(self, datasets, sa_session):
for dataset in datasets:
jeom = self._get_output_filenames_by_dataset(dataset, sa_session)
# shouldn't be more than one valid, but you never know
while jeom:
jeom.is_valid = False
sa_session.add(jeom)
sa_session.flush()
jeom = self._get_output_filenames_by_dataset(dataset, sa_session)
def setup_external_metadata(self, datasets_dict, out_collections, sa_session, exec_dir=None,
tmp_dir=None, dataset_files_path=None,
output_fnames=None, config_root=None, use_bin=False,
config_file=None, datatypes_config=None,
job_metadata=None, provided_metadata_style=None, compute_tmp_dir=None,
include_command=True, max_metadata_value_size=0,
validate_outputs=False,
object_store_conf=None, tool=None, job=None,
kwds=None):
kwds = kwds or {}
if not job:
job = sa_session.query(galaxy.model.Job).get(self.job_id)
tmp_dir = _init_tmp_dir(tmp_dir)
_assert_datatypes_config(datatypes_config)
# path is calculated for Galaxy, may be different on compute - rewrite
# for the compute server.
def metadata_path_on_compute(path):
compute_path = path
if compute_tmp_dir and tmp_dir and in_directory(path, tmp_dir):
path_relative = os.path.relpath(path, tmp_dir)
compute_path = os.path.join(compute_tmp_dir, path_relative)
return compute_path
# fill in metadata_files_dict and return the command with args required to set metadata
def __metadata_files_list_to_cmd_line(metadata_files):
line = '"{},{},{},{},{},{}"'.format(
metadata_path_on_compute(metadata_files.filename_in),
metadata_path_on_compute(metadata_files.filename_kwds),
metadata_path_on_compute(metadata_files.filename_out),
metadata_path_on_compute(metadata_files.filename_results_code),
_get_filename_override(output_fnames, metadata_files.dataset.file_name),
metadata_path_on_compute(metadata_files.filename_override_metadata),
)
return line
datasets = list(datasets_dict.values())
if exec_dir is None:
exec_dir = os.path.abspath(os.getcwd())
if dataset_files_path is None:
dataset_files_path = galaxy.model.Dataset.file_path
if config_root is None:
config_root = os.path.abspath(os.getcwd())
metadata_files_list = []
for dataset in datasets:
key = self._get_dataset_metadata_key(dataset)
# future note:
# wonkiness in job execution causes build command line to be called more than once
# when setting metadata externally, via 'auto-detect' button in edit attributes, etc.,
# we don't want to overwrite (losing the ability to cleanup) our existing dataset keys and files,
# so we will only populate the dictionary once
metadata_files = self._get_output_filenames_by_dataset(dataset, sa_session)
if not metadata_files:
metadata_files = galaxy.model.JobExternalOutputMetadata(job=job, dataset=dataset)
# we are using tempfile to create unique filenames, tempfile always returns an absolute path
# we will use pathnames relative to the galaxy root, to accommodate instances where the galaxy root
# is located differently, i.e. on a cluster node with a different filesystem structure
def _metadata_path(what):
return abspath(tempfile.NamedTemporaryFile(dir=tmp_dir, prefix=f"metadata_{what}_{key}_").name)
filename_in, filename_out, filename_results_code, filename_kwds, filename_override_metadata = _initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds)
# file to store existing dataset
metadata_files.filename_in = filename_in
# file to store metadata results of set_meta()
metadata_files.filename_out = filename_out
# file to store a 'return code' indicating the results of the set_meta() call
# results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure )
metadata_files.filename_results_code = filename_results_code
# file to store kwds passed to set_meta()
metadata_files.filename_kwds = filename_kwds
# existing metadata file parameters need to be overridden with cluster-writable file locations
metadata_files.filename_override_metadata = filename_override_metadata
# add to session and flush
sa_session.add(metadata_files)
sa_session.flush()
metadata_files_list.append(metadata_files)
args = '"{}" "{}" {} {}'.format(metadata_path_on_compute(datatypes_config),
job_metadata,
" ".join(map(__metadata_files_list_to_cmd_line, metadata_files_list)),
max_metadata_value_size)
assert not use_bin
if include_command:
# return command required to build
with tempfile.NamedTemporaryFile(mode='w', suffix='.py', dir=tmp_dir, prefix="set_metadata_", delete=False) as temp:
temp.write(SET_METADATA_SCRIPT)
return 'python "{}" {}'.format(metadata_path_on_compute(temp.name), args)
else:
# return args to galaxy_ext.metadata.set_metadata required to build
return args
def external_metadata_set_successfully(self, dataset, name, sa_session, working_directory):
metadata_files = self._get_output_filenames_by_dataset(dataset, sa_session)
if not metadata_files:
return False # this file doesn't exist
return self._metadata_results_from_file(dataset, metadata_files.filename_results_code)
def cleanup_external_metadata(self, sa_session):
log.debug('Cleaning up external metadata files')
for metadata_files in sa_session.query(galaxy.model.Job).get(self.job_id).external_output_metadata:
# we need to confirm that any MetadataTempFile files were removed, if not we need to remove them
# can occur if the job was stopped before completion, but a MetadataTempFile is used in the set_meta
MetadataTempFile.cleanup_from_JSON_dict_filename(metadata_files.filename_out)
dataset_key = self._get_dataset_metadata_key(metadata_files.dataset)
for key, fname in [('filename_in', metadata_files.filename_in),
('filename_out', metadata_files.filename_out),
('filename_results_code', metadata_files.filename_results_code),
('filename_kwds', metadata_files.filename_kwds),
('filename_override_metadata', metadata_files.filename_override_metadata)]:
try:
os.remove(fname)
except Exception as e:
log.debug(f'Failed to cleanup external metadata file ({key}) for {dataset_key}: {e}')
def set_job_runner_external_pid(self, pid, sa_session):
for metadata_files in sa_session.query(galaxy.model.Job).get(self.job_id).external_output_metadata:
metadata_files.job_runner_external_pid = pid
sa_session.add(metadata_files)
sa_session.flush()
def load_metadata(self, dataset, name, sa_session, working_directory, remote_metadata_directory=None):
# load metadata from file
# we need to no longer allow metadata to be edited while the job is still running,
# since if it is edited, the metadata changed on the running output will no longer match
# the metadata that was stored to disk for use via the external process,
# and the changes made by the user will be lost, without warning or notice
output_filename = self._get_output_filenames_by_dataset(dataset, sa_session).filename_out
self._load_metadata_from_path(dataset, output_filename, working_directory, remote_metadata_directory)
def _initialize_metadata_inputs(dataset, path_for_part, tmp_dir, kwds, real_metadata_object=True):
filename_in = path_for_part("in")
filename_out = path_for_part("out")
filename_results_code = path_for_part("results")
filename_kwds = path_for_part("kwds")
filename_override_metadata = path_for_part("override")
_dump_dataset_instance_to(dataset, filename_in)
open(filename_out, 'wt+') # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
# create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
json.dump((False, 'External set_meta() not called'), open(filename_results_code, 'wt+'))
@@ -446,26 +259,7 @@ def _initialize_metadata_inputs(dataset, path_for_part, tmp_dir, kwds, real_meta
json.dump(override_metadata, open(filename_override_metadata, 'wt+'))
return filename_in, filename_out, filename_results_code, filename_kwds, filename_override_metadata
def _assert_datatypes_config(datatypes_config):
if datatypes_config is None:
raise Exception('In setup_external_metadata, the received datatypes_config is None.')
def _dump_dataset_instance_to(dataset_instance, file_path):
# FIXME: HACK
# sqlalchemy introduced 'expire_on_commit' flag for sessionmaker at version 0.5x
# This may be causing the dataset attribute of the dataset_association object to no-longer be loaded into memory when needed for pickling.
# For now, we'll simply 'touch' dataset_association.dataset to force it back into memory.
dataset_instance.dataset # force dataset_association.dataset to be loaded before pickling
# A better fix could be setting 'expire_on_commit=False' on the session, or modifying where commits occur, or ?
# Touch also deferred column
dataset_instance._metadata
pickle.dump(dataset_instance, open(file_path, 'wb+'))
return filename_out, filename_results_code, filename_kwds, filename_override_metadata
def _get_filename_override(output_fnames, file_name):
lib/galaxy/metadata/set_metadata.py
+60 -104
View File
@@ -3,7 +3,7 @@ Execute an external process to set_meta() on a provided list of pickled datasets
This was formerly scripts/set_metadata.py and expects these arguments:
%prog datatypes_conf.xml job_metadata_file metadata_in,metadata_kwds,metadata_out,metadata_results_code,output_filename_override,metadata_override... max_metadata_value_size
%prog datatypes_conf.xml job_metadata_file metadata_kwds,metadata_out,metadata_results_code,output_filename_override,metadata_override... max_metadata_value_size
Galaxy should be importable on sys.path and output_filename_override should be
set to the path of the dataset on which metadata is being set
@@ -13,27 +13,55 @@ constructed automatically).
import json
import logging
import os
import pickle
import sys
import traceback
from sqlalchemy.orm import clear_mappers
try:
from pulsar.client.staging import COMMAND_VERSION_FILENAME
except ImportError:
# Package unit tests
COMMAND_VERSION_FILENAME = 'COMMAND_VERSION'
import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers
from galaxy.model import store
from galaxy.model.custom_types import total_size
from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
from galaxy.util import (
stringify_dictionary_keys,
unicodify,
import galaxy.datatypes.registry
import galaxy.model.mapping
from galaxy.datatypes import sniff
from galaxy.datatypes.data import validate
from galaxy.job_execution.output_collect import (
collect_dynamic_outputs,
collect_extra_files,
collect_primary_datasets,
default_exit_code_file,
read_exit_code_from,
SessionlessJobContext,
)
from galaxy.job_execution.setup import TOOL_PROVIDED_JOB_METADATA_KEYS
from galaxy.model import (
Dataset,
HistoryDatasetAssociation,
HistoryDatasetCollectionAssociation,
Job,
store,
)
from galaxy.model.custom_types import total_size
from galaxy.model.metadata import MetadataTempFile
from galaxy.objectstore import build_object_store_from_config
from galaxy.tool_util.output_checker import (
check_output,
DETECTED_JOB_STATE,
)
from galaxy.tool_util.parser.stdio import (
ToolStdioExitCode,
ToolStdioRegex,
)
from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
from galaxy.util import stringify_dictionary_keys
from galaxy.util.expressions import ExpressionContext
logging.basicConfig()
log = logging.getLogger(__name__)
def set_validated_state(dataset_instance):
from galaxy.datatypes.data import validate
datatype_validation = validate(dataset_instance)
dataset_instance.validated_state = datatype_validation.state
@@ -53,7 +81,6 @@ def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, data
extension = dataset_instance.extension
if extension == "_sniff_":
try:
from galaxy.datatypes import sniff
extension = sniff.handle_uploaded_dataset_file(dataset_instance.dataset.external_filename, datatypes_registry)
# We need to both set the extension so it is available to set_meta
# and record it in the metadata so it can be reloaded on the server
@@ -78,17 +105,13 @@ def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, data
def set_metadata():
if len(sys.argv) == 1:
set_metadata_portable()
else:
set_metadata_legacy()
set_metadata_portable()
def set_metadata_portable():
import galaxy.model
tool_job_working_directory = os.path.abspath(os.getcwd())
metadata_tmp_files_dir = os.path.join(tool_job_working_directory, "metadata")
galaxy.model.metadata.MetadataTempFile.tmp_dir = metadata_tmp_files_dir
MetadataTempFile.tmp_dir = metadata_tmp_files_dir
metadata_params_path = os.path.join("metadata", "params.json")
try:
@@ -117,7 +140,6 @@ def set_metadata_portable():
export_store = None
if extended_metadata_collection:
from galaxy.tool_util.parser.stdio import ToolStdioRegex, ToolStdioExitCode
tool_dict = metadata_params["tool"]
stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"]
stdio_exit_codes = list(map(ToolStdioExitCode, stdio_exit_code_dicts))
@@ -125,10 +147,9 @@ def set_metadata_portable():
with open(object_store_conf_path) as f:
config_dict = json.load(f)
from galaxy.objectstore import build_object_store_from_config
assert config_dict is not None
object_store = build_object_store_from_config(None, config_dict=config_dict)
galaxy.model.Dataset.object_store = object_store
Dataset.object_store = object_store
outputs_directory = os.path.join(tool_job_working_directory, "outputs")
if not os.path.exists(outputs_directory):
@@ -151,38 +172,35 @@ def set_metadata_portable():
job_id_tag = metadata_params["job_id_tag"]
# TODO: this clearly needs to be refactored, nothing in runners should be imported here..
from galaxy.job_execution.output_collect import default_exit_code_file, read_exit_code_from
exit_code_file = default_exit_code_file(".", job_id_tag)
tool_exit_code = read_exit_code_from(exit_code_file, job_id_tag)
from galaxy.tool_util.output_checker import check_output, DETECTED_JOB_STATE
check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output(stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag)
if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs():
final_job_state = galaxy.model.Job.states.OK
final_job_state = Job.states.OK
else:
final_job_state = galaxy.model.Job.states.ERROR
final_job_state = Job.states.ERROR
from pulsar.client.staging import COMMAND_VERSION_FILENAME
version_string = ""
if os.path.exists(COMMAND_VERSION_FILENAME):
version_string = open(COMMAND_VERSION_FILENAME).read()
from galaxy.util.expressions import ExpressionContext
job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
# Load outputs.
import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store)
export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False)
import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store)
for output_name, output_dict in outputs.items():
if extended_metadata_collection:
dataset_instance_id = output_dict["id"]
dataset = import_model_store.sa_session.query(galaxy.model.HistoryDatasetAssociation).find(dataset_instance_id)
assert dataset is not None
else:
dataset_instance_id = output_dict["id"]
klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation'))
dataset = import_model_store.sa_session.query(klass).find(dataset_instance_id)
if dataset is None:
# legacy check for jobs that started before 21.01, remove on 21.05
filename_in = os.path.join("metadata/metadata_in_%s" % output_name)
import pickle
dataset = pickle.load(open(filename_in, 'rb')) # load DatasetInstance
assert dataset is not None
filename_kwds = os.path.join("metadata/metadata_kwds_%s" % output_name)
filename_out = os.path.join("metadata/metadata_out_%s" % output_name)
@@ -206,8 +224,8 @@ def set_metadata_portable():
# Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles
override_metadata = json.load(open(override_metadata))
for metadata_name, metadata_file_override in override_metadata:
if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override):
metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override)
if MetadataTempFile.is_JSONified_value(metadata_file_override):
metadata_file_override = MetadataTempFile.from_JSON(metadata_file_override)
setattr(dataset.metadata, metadata_name, metadata_file_override)
if output_dict.get("validate", False):
set_validated_state(dataset)
@@ -240,9 +258,8 @@ def set_metadata_portable():
# This has to be a job with outputs_to_working_directory set.
# We update the object store with the created output file.
object_store.update_from_file(dataset.dataset, file_name=dataset_filename_override, create=True)
from galaxy.job_execution.output_collect import collect_extra_files
collect_extra_files(object_store, dataset, ".")
if galaxy.model.Job.states.ERROR == final_job_state:
if Job.states.ERROR == final_job_state:
dataset.blurb = "error"
dataset.mark_unhidden()
else:
@@ -262,7 +279,6 @@ def set_metadata_portable():
# ... and others don't
dataset.set_peek()
from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_KEYS
for context_key in TOOL_PROVIDED_JOB_METADATA_KEYS:
if context_key in context:
context_value = context[context_key]
@@ -279,7 +295,6 @@ def set_metadata_portable():
if extended_metadata_collection:
# discover extra outputs...
from galaxy.job_execution.output_collect import collect_dynamic_outputs, collect_primary_datasets, SessionlessJobContext
job_context = SessionlessJobContext(
metadata_params,
@@ -293,10 +308,11 @@ def set_metadata_portable():
output_collections = {}
for name, output_collection in metadata_params["output_collections"].items():
output_collections[name] = import_model_store.sa_session.query(galaxy.model.HistoryDatasetCollectionAssociation).find(output_collection["id"])
output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"])
outputs = {}
for name, output in metadata_params["outputs"].items():
outputs[name] = import_model_store.sa_session.query(galaxy.model.HistoryDatasetAssociation).find(output["id"])
klass = getattr(galaxy.model, output.get('model_class', 'HistoryDatasetAssociation'))
outputs[name] = import_model_store.sa_session.query(klass).find(output["id"])
input_ext = json.loads(metadata_params["job_params"].get("__input_ext", '"data"'))
collect_primary_datasets(
@@ -311,64 +327,6 @@ def set_metadata_portable():
write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata)
def set_metadata_legacy():
import galaxy.model
galaxy.model.metadata.MetadataTempFile.tmp_dir = tool_job_working_directory = os.path.abspath(os.getcwd())
# This is ugly, but to transition from existing jobs without this parameter
# to ones with, smoothly, it has to be the last optional parameter and we
# have to sniff it.
try:
max_metadata_value_size = int(sys.argv[-1])
sys.argv = sys.argv[:-1]
except ValueError:
max_metadata_value_size = 0
# max_metadata_value_size is unspecified and should be 0
# Set up datatypes registry
datatypes_config = sys.argv.pop(1)
datatypes_registry = validate_and_load_datatypes_config(datatypes_config)
job_metadata = sys.argv.pop(1)
tool_provided_metadata = load_job_metadata(job_metadata, None)
def set_meta(new_dataset_instance, file_dict):
set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size)
for filenames in sys.argv[1:]:
fields = filenames.split(',')
filename_in = fields.pop(0)
filename_kwds = fields.pop(0)
filename_out = fields.pop(0)
filename_results_code = fields.pop(0)
dataset_filename_override = fields.pop(0)
override_metadata = fields.pop(0)
set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode
try:
dataset = pickle.load(open(filename_in, 'rb')) # load DatasetInstance
dataset.dataset.external_filename = dataset_filename_override
store_by = "id"
extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by)
files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name))
dataset.dataset.external_extra_files_path = files_path
file_dict = tool_provided_metadata.get_dataset_meta(None, dataset.dataset.id, dataset.dataset.uuid)
if 'ext' in file_dict:
dataset.extension = file_dict['ext']
# Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles
override_metadata = json.load(open(override_metadata))
for metadata_name, metadata_file_override in override_metadata:
if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override):
metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override)
setattr(dataset.metadata, metadata_name, metadata_file_override)
set_meta(dataset, file_dict)
dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta
json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded
except Exception as e:
json.dump((False, unicodify(e)), open(filename_results_code, 'wt+')) # setting metadata has failed somehow
write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata)
def validate_and_load_datatypes_config(datatypes_config):
galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir))
@@ -379,7 +337,6 @@ def validate_and_load_datatypes_config(datatypes_config):
if not os.path.exists(datatypes_config):
print("Metadata setting failed because registry.xml [%s] could not be found. You may retry setting metadata." % datatypes_config)
sys.exit(1)
import galaxy.datatypes.registry
datatypes_registry = galaxy.datatypes.registry.Registry()
datatypes_registry.load_datatypes(root_dir=galaxy_root, config=datatypes_config, use_build_sites=False, use_converters=False, use_display_applications=False)
galaxy.model.set_datatypes_registry(datatypes_registry)
@@ -394,14 +351,13 @@ def write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_
for i, file_dict in enumerate(tool_provided_metadata.get_new_datasets_for_metadata_collection(), start=1):
filename = file_dict["filename"]
new_dataset_filename = os.path.join(tool_job_working_directory, "working", filename)
new_dataset = galaxy.model.Dataset(id=-i, external_filename=new_dataset_filename)
new_dataset = Dataset(id=-i, external_filename=new_dataset_filename)
extra_files = file_dict.get('extra_files', None)
if extra_files is not None:
new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files)
new_dataset.state = new_dataset.states.OK
new_dataset_instance = galaxy.model.HistoryDatasetAssociation(id=-i, dataset=new_dataset, extension=file_dict.get('ext', 'data'))
new_dataset_instance = HistoryDatasetAssociation(id=-i, dataset=new_dataset, extension=file_dict.get('ext', 'data'))
set_meta(new_dataset_instance, file_dict)
file_dict['metadata'] = json.loads(new_dataset_instance.metadata.to_JSON_dict()) # storing metadata in external form, need to turn back into dict, then later jsonify
tool_provided_metadata.rewrite()
clear_mappers()
lib/galaxy/model/__init__.py
+1 -1
View File
@@ -4139,7 +4139,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
select_stmt = select(list(map(lambda dc: dc.c.populated_state, collection_depth_aliases))).select_from(select_from).where(dc.c.id == self.id).distinct()
for populated_states in db_session.execute(select_stmt).fetchall():
for populated_state in populated_states:
if populated_state != DatasetCollection.populated_states.OK:
if populated_state and populated_state != DatasetCollection.populated_states.OK:
_populated_optimized = False
self._populated_optimized = _populated_optimized
lib/galaxy/model/custom_types.py
+11 -141
View File
@@ -9,13 +9,15 @@ from sys import getsizeof
import numpy
import sqlalchemy
from sqlalchemy.ext.mutable import Mutable
from sqlalchemy.types import (
CHAR,
LargeBinary,
String,
TypeDecorator
)
from sqlalchemy_mutable.mutable import Mutable
# For compatibility with custom yaml dumping in gxformat2
from sqlalchemy_mutable.mutable_dict import MutableDict as MutationDict # noqa: F401
from galaxy.util import (
smart_str,
@@ -77,7 +79,7 @@ class GalaxyLargeBinary(LargeBinary):
return process
class JSONType(sqlalchemy.types.TypeDecorator):
class BaseJSONType(sqlalchemy.types.TypeDecorator):
"""
Represents an immutable structure as a json-encoded string.
@@ -113,149 +115,17 @@ class JSONType(sqlalchemy.types.TypeDecorator):
return (x == y)
class MutationObj(Mutable):
"""
Mutable JSONType for SQLAlchemy from original gist:
https://gist.github.com/dbarnett/1730610
Using minor changes from this fork of the gist:
https://gist.github.com/miracle2k/52a031cced285ba9b8cd
And other minor changes to make it work for us.
"""
@classmethod
def coerce(cls, key, value):
if isinstance(value, dict) and not isinstance(value, MutationDict):
return MutationDict.coerce(key, value)
if isinstance(value, list) and not isinstance(value, MutationList):
return MutationList.coerce(key, value)
return value
@classmethod
def _listen_on_attribute(cls, attribute, coerce, parent_cls):
key = attribute.key
if parent_cls is not attribute.class_:
return
# rely on "propagate" here
parent_cls = attribute.class_
def load(state, *args):
val = state.dict.get(key, None)
if coerce and key not in state.unloaded:
val = cls.coerce(key, val)
state.dict[key] = val
if isinstance(val, cls):
val._parents[state.obj()] = key
def set(target, value, oldvalue, initiator):
if not isinstance(value, cls):
value = cls.coerce(key, value)
if isinstance(value, cls):
value._parents[target.obj()] = key
if isinstance(oldvalue, cls):
oldvalue._parents.pop(target.obj(), None)
return value
def pickle(state, state_dict):
val = state.dict.get(key, None)
if isinstance(val, cls):
if 'ext.mutable.values' not in state_dict:
state_dict['ext.mutable.values'] = []
state_dict['ext.mutable.values'].append(val)
def unpickle(state, state_dict):
if 'ext.mutable.values' in state_dict:
for val in state_dict['ext.mutable.values']:
val._parents[state.obj()] = key
sqlalchemy.event.listen(parent_cls, 'load', load, raw=True, propagate=True)
sqlalchemy.event.listen(parent_cls, 'refresh', load, raw=True, propagate=True)
sqlalchemy.event.listen(attribute, 'set', set, raw=True, retval=True, propagate=True)
sqlalchemy.event.listen(parent_cls, 'pickle', pickle, raw=True, propagate=True)
sqlalchemy.event.listen(parent_cls, 'unpickle', unpickle, raw=True, propagate=True)
class SimpleJSONType(BaseJSONType):
"""SQLAlchemy column type that does not track mutations to mutable data."""
pass
class MutationDict(MutationObj, dict):
@classmethod
def coerce(cls, key, value):
"""Convert plain dictionary to MutationDict"""
self = MutationDict((k, MutationObj.coerce(key, v)) for (k, v) in value.items())
self._key = key
return self
def __setitem__(self, key, value):
if hasattr(self, '_key'):
value = MutationObj.coerce(self._key, value)
dict.__setitem__(self, key, value)
self.changed()
def __delitem__(self, key):
dict.__delitem__(self, key)
self.changed()
def __getstate__(self):
return dict(self)
def __setstate__(self, state):
self.update(state)
class JSONType(BaseJSONType):
"""SQLAlchemy column type that tracks mutations to mutable data."""
pass
class MutationList(MutationObj, list):
@classmethod
def coerce(cls, key, value):
"""Convert plain list to MutationList"""
self = MutationList(MutationObj.coerce(key, v) for v in value)
self._key = key
return self
def __setitem__(self, idx, value):
list.__setitem__(self, idx, MutationObj.coerce(self._key, value))
self.changed()
def __setslice__(self, start, stop, values):
list.__setslice__(self, start, stop, (MutationObj.coerce(self._key, v) for v in values))
self.changed()
def __delitem__(self, idx):
list.__delitem__(self, idx)
self.changed()
def __delslice__(self, start, stop):
list.__delslice__(self, start, stop)
self.changed()
def __copy__(self):
return MutationList(MutationObj.coerce(self._key, self[:]))
def __deepcopy__(self, memo):
return MutationList(MutationObj.coerce(self._key, copy.deepcopy(self[:])))
def append(self, value):
list.append(self, MutationObj.coerce(self._key, value))
self.changed()
def insert(self, idx, value):
list.insert(self, idx, MutationObj.coerce(self._key, value))
self.changed()
def extend(self, values):
if hasattr(self, '_key'):
values = (MutationObj.coerce(self._key, value) for value in values)
list.extend(self, values)
self.changed()
def pop(self, *args, **kw):
value = list.pop(self, *args, **kw)
self.changed()
return value
def remove(self, value):
list.remove(self, value)
self.changed()
MutationObj.associate_with(JSONType)
Mutable.associate_with(JSONType)
metadata_pickler = AliasPickleModule({
("cookbook.patterns", "Bunch"): ("galaxy.util.bunch", "Bunch")
lib/galaxy/model/dataset_collections/builder.py
+3 -5
View File
@@ -1,5 +1,3 @@
from collections import OrderedDict
from galaxy import model
from .type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY
@@ -34,7 +32,7 @@ class CollectionBuilder:
def __init__(self, collection_type_description):
self._collection_type_description = collection_type_description
self._current_elements = OrderedDict()
self._current_elements = {}
def replace_elements_in_collection(self, template_collection, replacement_dict):
self._current_elements = self._replace_elements_in_collection(
@@ -43,7 +41,7 @@ class CollectionBuilder:
)
def _replace_elements_in_collection(self, template_collection, replacement_dict):
elements = OrderedDict()
elements = {}
for element in template_collection.elements:
if element.is_collection:
collection_builder = CollectionBuilder(
@@ -77,7 +75,7 @@ class CollectionBuilder:
def build_elements(self):
elements = self._current_elements
if self._nested_collection:
new_elements = OrderedDict()
new_elements = {}
for identifier, element in elements.items():
new_elements[identifier] = element.build()
elements = new_elements
lib/galaxy/model/mapping.py
+11 -5
View File
@@ -40,7 +40,13 @@ from sqlalchemy.types import BigInteger
from galaxy import model
from galaxy.model.base import ModelMapping
from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString, UUIDType
from galaxy.model.custom_types import (
JSONType,
MetadataType,
SimpleJSONType,
TrimmedString,
UUIDType,
)
from galaxy.model.orm.engine_factory import build_engine
from galaxy.model.orm.now import now
from galaxy.model.security import GalaxyRBACAgent
@@ -860,7 +866,7 @@ model.PostJobAction.table = Table(
Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=True),
Column("action_type", String(255), nullable=False),
Column("output_name", String(255), nullable=True),
Column("action_arguments", JSONType, nullable=True))
Column("action_arguments", SimpleJSONType, nullable=True))
model.PostJobActionAssociation.table = Table(
"post_job_action_association", metadata,
@@ -1037,7 +1043,7 @@ model.WorkflowStepInput.table = Table(
Column("scatter_type", TEXT),
Column("value_from", JSONType),
Column("value_from_type", TEXT),
Column("default_value", JSONType),
Column("default_value", SimpleJSONType),
Column("default_value_set", Boolean, default=False),
Column("runtime_value", Boolean, default=False),
Index('ix_workflow_step_input_workflow_step_id_name_unique', "workflow_step_id", "name", unique=True, mysql_length={'name': 200}),
@@ -1066,7 +1072,7 @@ model.WorkflowRequestInputStepParameter.table = Table(
Column("id", Integer, primary_key=True),
Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True),
Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")),
Column("parameter_value", JSONType),
Column("parameter_value", SimpleJSONType),
)
model.WorkflowRequestToInputDatasetAssociation.table = Table(
@@ -1125,7 +1131,7 @@ model.WorkflowInvocationStep.table = Table(
Column("state", TrimmedString(64), index=True),
Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=True),
Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True),
Column("action", JSONType, nullable=True))
Column("action", SimpleJSONType, nullable=True))
model.WorkflowInvocationOutputDatasetAssociation.table = Table(
"workflow_invocation_output_dataset_association", metadata,
lib/galaxy/model/store/__init__.py
+30 -22
View File
@@ -370,7 +370,11 @@ class ModelImportStore(metaclass=abc.ABCMeta):
assert 'id' in dataset_attrs
object_import_tracker.hdas_by_id[dataset_attrs['id']] = dataset_instance
else:
object_import_tracker.lddas_by_key[dataset_attrs[object_key]] = dataset_instance
if object_key in dataset_attrs:
object_import_tracker.lddas_by_key[dataset_attrs[object_key]] = dataset_instance
else:
assert 'id' in dataset_attrs
object_import_tracker.lddas_by_key[dataset_attrs['id']] = dataset_instance
def _import_libraries(self, object_import_tracker):
object_key = self.object_key
@@ -486,29 +490,32 @@ class ModelImportStore(metaclass=abc.ABCMeta):
return dc
for collection_attrs in collections_attrs:
dc = import_collection(collection_attrs["collection"])
if 'id' in collection_attrs and self.import_options.allow_edit and not self.sessionless:
hdca = self.sa_session.query(model.HistoryDatasetCollectionAssociation).get(collection_attrs["id"])
# TODO: edit attributes...
else:
hdca = model.HistoryDatasetCollectionAssociation(collection=dc,
visible=True,
name=collection_attrs['display_name'],
implicit_output_name=collection_attrs.get("implicit_output_name"))
self._attach_raw_id_if_editing(hdca, collection_attrs)
hdca.history = history
if new_history and self.trust_hid(collection_attrs):
hdca.hid = collection_attrs['hid']
if 'collection' in collection_attrs:
dc = import_collection(collection_attrs["collection"])
if 'id' in collection_attrs and self.import_options.allow_edit and not self.sessionless:
hdca = self.sa_session.query(model.HistoryDatasetCollectionAssociation).get(collection_attrs["id"])
# TODO: edit attributes...
else:
object_import_tracker.requires_hid.append(hdca)
hdca = model.HistoryDatasetCollectionAssociation(collection=dc,
visible=True,
name=collection_attrs['display_name'],
implicit_output_name=collection_attrs.get("implicit_output_name"))
self._attach_raw_id_if_editing(hdca, collection_attrs)
self._session_add(hdca)
if object_key in collection_attrs:
object_import_tracker.hdcas_by_key[collection_attrs[object_key]] = hdca
hdca.history = history
if new_history and self.trust_hid(collection_attrs):
hdca.hid = collection_attrs['hid']
else:
object_import_tracker.requires_hid.append(hdca)
self._session_add(hdca)
if object_key in collection_attrs:
object_import_tracker.hdcas_by_key[collection_attrs[object_key]] = hdca
else:
assert 'id' in collection_attrs
object_import_tracker.hdcas_by_id[collection_attrs['id']] = hdca
else:
assert 'id' in collection_attrs
object_import_tracker.hdcas_by_id[collection_attrs['id']] = hdca
import_collection(collection_attrs)
def _attach_raw_id_if_editing(self, obj, attrs):
if self.sessionless and 'id' in attrs and self.import_options.allow_edit:
@@ -1251,7 +1258,8 @@ class DirectoryModelExportStore(ModelExportStore):
def record_associated_jobs(obj):
# Get the job object.
job = None
for assoc in obj.creating_job_associations:
for assoc in getattr(obj, 'creating_job_associations', []):
# For mapped over jobs obj could be DatasetCollection, which has no creating_job_association
job = assoc.job
break
if not job:
lib/galaxy/model/store/discover.py
+1 -2
View File
@@ -10,7 +10,6 @@ import logging
import os
from collections import (
namedtuple,
OrderedDict
)
from typing import Any, NamedTuple, Optional
@@ -517,7 +516,7 @@ def persist_target_to_export_store(target_dict, export_store, object_store, work
def persist_elements_to_hdca(model_persistence_context, elements, hdca, collector=None):
filenames = OrderedDict()
filenames = {}
def add_to_discovered_files(elements, parent_identifiers=None):
parent_identifiers = parent_identifiers or []
lib/galaxy/objectstore/__init__.py
+9 -4
View File
@@ -12,7 +12,6 @@ import random
import shutil
import threading
import time
from collections import OrderedDict
import yaml
@@ -183,6 +182,9 @@ class ObjectStore(metaclass=abc.ABCMeta):
To accommodate nested objectstores, obj is passed in so this metadata can
be returned for the ConcreteObjectStore corresponding to the object.
If the dataset is in a new or discarded state and an object_store_id has not
yet been set, this may return ``None``.
"""
@abc.abstractmethod
@@ -191,6 +193,9 @@ class ObjectStore(metaclass=abc.ABCMeta):
To accommodate nested objectstores, obj is passed in so this metadata can
be returned for the ConcreteObjectStore corresponding to the object.
If the dataset is in a new or discarded state and an object_store_id has not
yet been set, this may return ``None``.
"""
@abc.abstractmethod
@@ -701,10 +706,10 @@ class NestedObjectStore(BaseObjectStore):
return self._call_method('_get_object_url', obj, None, False, **kwargs)
def _get_concrete_store_name(self, obj):
return self._call_method('_get_concrete_store_name', obj, None, True)
return self._call_method('_get_concrete_store_name', obj, None, False)
def _get_concrete_store_description_markdown(self, obj):
return self._call_method('_get_concrete_store_description_markdown', obj, None, True)
return self._call_method('_get_concrete_store_description_markdown', obj, None, False)
def _get_store_by(self, obj):
return self._call_method('_get_store_by', obj, None, False)
@@ -932,7 +937,7 @@ class HierarchicalObjectStore(NestedObjectStore):
"""The default constructor. Extends `NestedObjectStore`."""
super().__init__(config, config_dict)
backends = OrderedDict()
backends = {}
for order, backend_def in enumerate(config_dict["backends"]):
backends[order] = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon)
lib/galaxy/openid/providers.py
+2 -3
View File
@@ -3,7 +3,6 @@ Contains OpenID provider functionality
"""
import logging
import os
from collections import OrderedDict
from galaxy.util import parse_xml, string_as_bool
@@ -115,7 +114,7 @@ class OpenIDProviders:
@classmethod
def from_elem(cls, xml_root):
oid_elem = xml_root
providers = OrderedDict()
providers = {}
for elem in oid_elem.findall('provider'):
try:
provider = OpenIDProvider.from_file(os.path.join('lib/galaxy/openid', elem.get('file')))
@@ -129,7 +128,7 @@ class OpenIDProviders:
if providers:
self.providers = providers
else:
self.providers = OrderedDict()
self.providers = {}
self._banned_identifiers = [provider.op_endpoint_url for provider in self.providers.values() if provider.never_associate_with_user]
def __iter__(self):
lib/galaxy/tool_shed/galaxy_install/migrate/check.py
+2 -3
View File
@@ -2,7 +2,6 @@ import logging
import os
import subprocess
import sys
from collections import OrderedDict
from migrate.versioning import repository, schema
from sqlalchemy import create_engine, MetaData, Table
@@ -34,7 +33,7 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options=None):
tool_shed_accessible = False
if app.new_installation:
# New installations will not be missing tools, so we don't need to worry about them.
missing_tool_configs_dict = OrderedDict()
missing_tool_configs_dict = {}
else:
tool_panel_configs = common_util.get_non_shed_tool_panel_configs(app)
if tool_panel_configs:
@@ -48,7 +47,7 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options=None):
# we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
# the database.
tool_shed_accessible = True
missing_tool_configs_dict = OrderedDict()
missing_tool_configs_dict = {}
have_tool_dependencies = False
for v in missing_tool_configs_dict.values():
if v:
lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py
+2 -3
View File
@@ -8,7 +8,6 @@ import os
import shutil
import tempfile
import threading
from collections import OrderedDict
from galaxy import util
from galaxy.tool_shed.galaxy_install import install_manager
@@ -113,7 +112,7 @@ class ToolMigrationManager:
# tool_shed_accessible to True so that the value of migrate_tools.version can
# be correctly set in the database.
tool_shed_accessible = True
missing_tool_configs_dict = OrderedDict()
missing_tool_configs_dict = {}
if tool_shed_accessible:
if len(self.proprietary_tool_confs) == 1:
plural = ''
@@ -387,7 +386,7 @@ class ToolMigrationManager:
entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process.
"""
tool_configs_to_filter = []
tool_panel_dict_for_display = OrderedDict()
tool_panel_dict_for_display = {}
if self.tool_path:
repo_install_dir = os.path.join(self.tool_path, relative_install_dir)
else:
lib/galaxy/tool_shed/tool_shed_registry.py
+3 -6
View File
@@ -1,8 +1,5 @@
import logging
from collections import (
namedtuple,
OrderedDict,
)
from collections import namedtuple
from galaxy.util import parse_xml_string
from galaxy.util.tool_shed.common_util import remove_protocol_from_tool_shed_url
@@ -22,8 +19,8 @@ AUTH_TUPLE = namedtuple('AuthSetting', 'username password')
class Registry:
def __init__(self, config=None):
self.tool_sheds = OrderedDict()
self.tool_sheds_auth = OrderedDict()
self.tool_sheds = {}
self.tool_sheds_auth = {}
if config:
# Parse tool_sheds_conf.xml
tree, error_message = parse_xml(config)
lib/galaxy/tool_util/cwl/parser.py
+1 -2
View File
@@ -11,7 +11,6 @@ import logging
import os
import pickle
from abc import ABCMeta, abstractmethod
from collections import OrderedDict
from uuid import uuid4
@@ -1214,7 +1213,7 @@ class ConditionalInstance:
name=self.name,
type=INPUT_TYPE.CONDITIONAL,
test=self.case.to_dict(),
when=OrderedDict(),
when={},
)
for value, block in self.whens:
as_dict["when"][value] = [i.to_dict() for i in block]
lib/galaxy/tool_util/cwl/representation.py
+1 -2
View File
@@ -1,7 +1,6 @@
""" This module is responsible for converting between Galaxy's tool
input description and the CWL description for a job json. """
import collections
import json
import logging
import os
@@ -210,7 +209,7 @@ def collection_wrapper_to_array(inputs_dir, wrapped_value):
def collection_wrapper_to_record(inputs_dir, wrapped_value):
rval = collections.OrderedDict()
rval = {}
for key, value in wrapped_value.items():
rval[key] = dataset_wrapper_to_file_json(inputs_dir, value)
return rval
lib/galaxy/tool_util/deps/__init__.py
+2 -3
View File
@@ -6,7 +6,6 @@ import json
import logging
import os.path
import shutil
from collections import OrderedDict
from galaxy.util import (
hash_util,
@@ -201,7 +200,7 @@ class DependencyManager:
def _requirements_to_dependencies_dict(self, requirements, search=False, **kwds):
"""Build simple requirements to dependencies dict for resolution."""
requirement_to_dependency = OrderedDict()
requirement_to_dependency = {}
index = kwds.get('index')
install = kwds.get('install', False)
resolver_type = kwds.get('resolver_type')
@@ -233,7 +232,7 @@ class DependencyManager:
if container_type is not None and getattr(resolver, "container_type", None) != container_type:
continue
_requirement_to_dependency = OrderedDict([(k, v) for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)])
_requirement_to_dependency = {k: v for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)}
if len(_requirement_to_dependency) == len(resolvable_requirements):
# Shortcut - resolution complete.
lib/galaxy/tool_util/parser/cwl.py
+2 -3
View File
@@ -1,6 +1,5 @@
import logging
import os
from collections import OrderedDict
from galaxy.tool_util.cwl.parser import tool_proxy
from galaxy.tool_util.deps import requirements
@@ -113,14 +112,14 @@ class CwlToolSource(ToolSource):
def parse_outputs(self, tool):
output_instances = self.tool_proxy.output_instances()
outputs = OrderedDict()
outputs = {}
output_defs = []
for output_instance in output_instances:
output_defs.append(self._parse_output(tool, output_instance))
# TODO: parse outputs collections
for output_def in output_defs:
outputs[output_def.name] = output_def
return outputs, OrderedDict()
return outputs, {}
def _parse_output(self, tool, output_instance):
name = output_instance.name
lib/galaxy/tool_util/parser/output_objects.py
+1 -3
View File
@@ -1,5 +1,3 @@
from collections import OrderedDict
from galaxy.util.dictifiable import Dictifiable
from .output_actions import ToolOutputActionGroup
from .output_collection_def import dataset_collector_descriptions_from_output_dict
@@ -156,7 +154,7 @@ class ToolOutputCollection(ToolOutputBase):
self.collection = True
self.default_format = default_format
self.structure = structure
self.outputs = OrderedDict()
self.outputs = {}
self.inherit_format = inherit_format
self.inherit_metadata = inherit_metadata
lib/galaxy/tool_util/parser/xml.py
+3 -4
View File
@@ -2,7 +2,6 @@ import json
import logging
import re
import uuid
from collections import OrderedDict
from math import isinf
import packaging.version
@@ -290,12 +289,12 @@ class XmlToolSource(ToolSource):
def parse_outputs(self, tool):
out_elem = self.root.find("outputs")
outputs = OrderedDict()
output_collections = OrderedDict()
outputs = {}
output_collections = {}
if out_elem is None:
return outputs, output_collections
data_dict = OrderedDict()
data_dict = {}
def _parse(data_elem, **kwds):
output_def = self._parse_output(data_elem, tool, **kwds)
lib/galaxy/tool_util/parser/yaml.py
+2 -4
View File
@@ -1,5 +1,3 @@
from collections import OrderedDict
import packaging.version
from galaxy.tool_util.deps import requirements
@@ -117,10 +115,10 @@ class YamlToolSource(ToolSource):
else:
message = "Unknown output_type [%s] encountered." % output_type
raise Exception(message)
outputs = OrderedDict()
outputs = {}
for output in output_defs:
outputs[output.name] = output
output_collections = OrderedDict()
output_collections = {}
for output in output_collection_defs:
output_collections[output.name] = output
lib/galaxy/tool_util/verify/interactor.py
+3 -9
View File
@@ -8,7 +8,6 @@ import tarfile
import tempfile
import time
import zipfile
from collections import OrderedDict
from json import dumps
from logging import getLogger
@@ -43,7 +42,7 @@ DEFAULT_FTYPE = 'auto'
DEFAULT_DBKEY = os.environ.get("GALAXY_TEST_DEFAULT_DBKEY", "?")
class OutputsDict(OrderedDict):
class OutputsDict(dict):
"""Ordered dict that can also be accessed by index.
>>> out = OutputsDict()
@@ -57,12 +56,7 @@ class OutputsDict(OrderedDict):
if isinstance(item, int):
return self[list(self.keys())[item]]
else:
# ideally we'd do `return super(OutputsDict, self)[item]`,
# but this fails because OrderedDict has no `__getitem__`. (!?)
item = self.get(item)
if item is None:
raise KeyError(item)
return item
return super().__getitem__(item)
def stage_data_in_history(galaxy_interactor, tool_id, all_test_data, history=None, force_path_paste=False, maxseconds=DEFAULT_TOOL_TEST_WAIT):
@@ -497,7 +491,7 @@ class GalaxyInteractorApi:
return element_identifiers
def __dictify_output_collections(self, submit_response):
output_collections_dict = OrderedDict()
output_collections_dict = {}
for output_collection in submit_response['output_collections']:
output_collections_dict[output_collection.get("output_name")] = output_collection
return output_collections_dict
lib/galaxy/tools/__init__.py
+18 -19
View File
@@ -10,7 +10,6 @@ import re
import tarfile
import tempfile
import threading
from collections import OrderedDict
from datetime import datetime
from pathlib import Path
from typing import List, Type
@@ -1124,7 +1123,7 @@ class Tool(Dictifiable):
This implementation supports multiple pages and grouping constructs.
"""
# Load parameters (optional)
self.inputs = OrderedDict()
self.inputs = {}
pages = tool_source.parse_input_pages()
enctypes = set()
if pages.inputs_defined:
@@ -1237,7 +1236,7 @@ class Tool(Dictifiable):
groups (repeat, conditional) or param elements. Groups will be parsed
recursively.
"""
rval = OrderedDict()
rval = {}
context = ExpressionContext(rval, context)
for input_source in page_source.parse_input_sources():
# Repeat group
@@ -1277,7 +1276,7 @@ class Tool(Dictifiable):
page_source = XmlPageSource(XML("<when>%s</when>" % case_inputs))
case.inputs = self.parse_input_elem(page_source, enctypes, context)
else:
case.inputs = OrderedDict()
case.inputs = {}
group.cases.append(case)
else:
# Should have one child "input" which determines the case
@@ -1305,7 +1304,7 @@ class Tool(Dictifiable):
(self.id, group.name, group.test_param.name, unspecified_case))
case = ConditionalWhen()
case.value = unspecified_case
case.inputs = OrderedDict()
case.inputs = {}
group.cases.append(case)
rval[group.name] = group
elif input_type == "section":
@@ -1673,7 +1672,7 @@ class Tool(Dictifiable):
log.exception("Exception caught while attempting to execute tool with id '%s':", self.id)
message = "Error executing tool with id '{}': {}".format(self.id, unicodify(e))
return False, message
if isinstance(out_data, OrderedDict):
if isinstance(out_data, dict):
return job, list(out_data.items())
else:
if isinstance(out_data, str):
@@ -2807,7 +2806,7 @@ class DatabaseOperationTool(Tool):
return self._outputs_dict()
def _outputs_dict(self):
return OrderedDict()
return {}
class UnzipCollectionTool(DatabaseOperationTool):
@@ -2839,7 +2838,7 @@ class ZipCollectionTool(DatabaseOperationTool):
reverse_o = incoming["input_reverse"]
forward, reverse = forward_o.copy(), reverse_o.copy()
new_elements = OrderedDict()
new_elements = {}
new_elements["forward"] = forward
new_elements["reverse"] = reverse
self._add_datasets_to_history(history, [forward, reverse])
@@ -2852,7 +2851,7 @@ class BuildListCollectionTool(DatabaseOperationTool):
tool_type = 'build_list'
def produce_outputs(self, trans, out_data, output_collections, incoming, history, tags=None, **kwds):
new_elements = OrderedDict()
new_elements = {}
for i, incoming_repeat in enumerate(incoming["datasets"]):
if incoming_repeat["input"]:
@@ -2912,7 +2911,7 @@ class MergeCollectionTool(DatabaseOperationTool):
if dupl_actions in ['suffix_conflict', 'suffix_every', 'suffix_conflict_rest']:
suffix_pattern = advanced['conflict']['suffix_pattern']
new_element_structure = OrderedDict()
new_element_structure = {}
# Which inputs does the identifier appear in.
identifiers_map = {}
@@ -2964,7 +2963,7 @@ class MergeCollectionTool(DatabaseOperationTool):
new_element_structure[effective_identifer] = element
# Don't copy until we know everything is fine and we have the structure of the list ready to go.
new_elements = OrderedDict()
new_elements = {}
for key, value in new_element_structure.items():
if getattr(value, "history_content_type", None) == "dataset":
copied_value = value.copy(flush=False)
@@ -2981,7 +2980,7 @@ class MergeCollectionTool(DatabaseOperationTool):
class FilterDatasetsTool(DatabaseOperationTool):
def _get_new_elements(self, history, elements_to_copy):
new_elements = OrderedDict()
new_elements = {}
for dce in elements_to_copy:
element_identifier = dce.element_identifier
if getattr(dce.element_object, "history_content_type", None) == "dataset":
@@ -3049,7 +3048,7 @@ class FlattenTool(DatabaseOperationTool):
def produce_outputs(self, trans, out_data, output_collections, incoming, history, **kwds):
hdca = incoming["input"]
join_identifier = incoming["join_identifier"]
new_elements = OrderedDict()
new_elements = {}
copied_datasets = []
def add_elements(collection, prefix=""):
@@ -3077,7 +3076,7 @@ class SortTool(DatabaseOperationTool):
def produce_outputs(self, trans, out_data, output_collections, incoming, history, **kwds):
hdca = incoming["input"]
sorttype = incoming["sort_type"]["sort_type"]
new_elements = OrderedDict()
new_elements = {}
elements = hdca.collection.elements
presort_elements = []
if sorttype == 'alpha':
@@ -3090,7 +3089,7 @@ class SortTool(DatabaseOperationTool):
hda = incoming["sort_type"]["sort_file"]
data_lines = hda.metadata.get('data_lines', 0)
if data_lines == len(elements):
old_elements_dict = OrderedDict()
old_elements_dict = {}
for element in elements:
old_elements_dict[element.element_identifier] = element
try:
@@ -3123,7 +3122,7 @@ class RelabelFromFileTool(DatabaseOperationTool):
how_type = incoming["how"]["how_select"]
new_labels_dataset_assoc = incoming["how"]["labels"]
strict = string_as_bool(incoming["how"]["strict"])
new_elements = OrderedDict()
new_elements = {}
def add_copied_value_to_new_elements(new_label, dce_object):
new_label = new_label.strip()
@@ -3199,7 +3198,7 @@ class TagFromFileTool(DatabaseOperationTool):
hdca = incoming["input"]
how = incoming['how']
new_tags_dataset_assoc = incoming["tags"]
new_elements = OrderedDict()
new_elements = {}
new_datasets = []
def add_copied_value_to_new_elements(new_tags_dict, dce):
@@ -3260,8 +3259,8 @@ class FilterFromFileTool(DatabaseOperationTool):
hdca = incoming["input"]
how_filter = incoming["how"]["how_filter"]
filter_dataset_assoc = incoming["how"]["filter_source"]
filtered_elements = OrderedDict()
discarded_elements = OrderedDict()
filtered_elements = {}
discarded_elements = {}
filtered_path = filter_dataset_assoc.file_name
with open(filtered_path) as fh:
lib/galaxy/tools/actions/__init__.py
+3 -4
View File
@@ -2,7 +2,6 @@ import json
import logging
import os
import re
from collections import OrderedDict
from json import dumps
@@ -69,7 +68,7 @@ class DefaultToolAction:
"""
if current_user_roles is None:
current_user_roles = trans.get_current_user_roles()
input_datasets = OrderedDict()
input_datasets = {}
all_permissions = {}
def record_permission(action, role_id):
@@ -357,7 +356,7 @@ class DefaultToolAction:
# wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
wrapped_params = self._wrapped_params(trans, tool, incoming, inp_data)
out_data = OrderedDict()
out_data = {}
input_collections = {k: v[0][0] for k, v in inp_dataset_collections.items()}
output_collections = OutputCollections(
trans,
@@ -852,7 +851,7 @@ class OutputCollections:
# We don't care about the repeat index, we just need to find the correct DataCollectionToolParameter
else:
key = group
if isinstance(data_param, OrderedDict):
if isinstance(data_param, dict):
data_param = data_param.get(key)
else:
data_param = data_param.inputs.get(key)
lib/galaxy/tools/actions/history_imp_exp.py
+2 -3
View File
@@ -2,7 +2,6 @@ import datetime
import logging
import os
import tempfile
from collections import OrderedDict
from galaxy.job_execution.setup import create_working_directory_for_job
from galaxy.tools.actions import ToolAction
@@ -73,7 +72,7 @@ class ImportHistoryToolAction(ToolAction):
trans.app.job_manager.enqueue(job, tool=tool)
trans.log_event("Added import history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
return job, OrderedDict()
return job, {}
class ExportHistoryToolAction(ToolAction):
@@ -173,4 +172,4 @@ class ExportHistoryToolAction(ToolAction):
trans.app.job_manager.enqueue(job, tool=tool)
trans.log_event("Added export history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
return job, OrderedDict()
return job, {}
lib/galaxy/tools/actions/metadata.py
+1 -2
View File
@@ -1,6 +1,5 @@
import logging
import os
from collections import OrderedDict
from json import dumps
from galaxy.job_execution.datasets import DatasetPath
@@ -129,4 +128,4 @@ class SetMetadataToolAction(ToolAction):
# clear e.g. converted files
dataset.datatype.before_setting_metadata(dataset)
return job, OrderedDict()
return job, {}
lib/galaxy/tools/actions/model_operations.py
+1 -2
View File
@@ -1,5 +1,4 @@
import logging
from collections import OrderedDict
from galaxy.tools.actions import (
DefaultToolAction,
@@ -38,7 +37,7 @@ class ModelOperationToolAction(DefaultToolAction):
# wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
wrapped_params = self._wrapped_params(trans, tool, incoming)
out_data = OrderedDict()
out_data = {}
input_collections = {k: v[0][0] for k, v in inp_dataset_collections.items()}
output_collections = OutputCollections(
trans,
lib/galaxy/tools/actions/upload_common.py
+1 -2
View File
@@ -3,7 +3,6 @@ import logging
import os
import socket
import tempfile
from collections import OrderedDict
from io import StringIO
from json import dump, dumps
from urllib.parse import urlparse
@@ -442,7 +441,7 @@ def create_job(trans, params, tool, json_file_path, outputs, folder=None, histor
# Queue the job for execution
trans.app.job_manager.enqueue(job, tool=tool)
trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
output = OrderedDict()
output = {}
for i, v in enumerate(outputs):
if not hasattr(output_object, "collection_type"):
output['output%i' % i] = v
lib/galaxy/tools/data/__init__.py
+1 -2
View File
@@ -14,7 +14,6 @@ import os.path
import re
import string
import time
from collections import OrderedDict
from glob import glob
from tempfile import NamedTemporaryFile
from typing import List
@@ -256,7 +255,7 @@ class ToolDataTable:
self.empty_field_values = {}
self.allow_duplicate_entries = util.asbool(config_element.get('allow_duplicate_entries', True))
self.here = filename and os.path.dirname(filename)
self.filenames = OrderedDict()
self.filenames = {}
self.tool_data_path = tool_data_path
self.tool_data_path_files = tool_data_path_files
self.other_config_dict = other_config_dict or {}
lib/galaxy/tools/data_manager/manager.py
+4 -5
View File
@@ -2,7 +2,6 @@ import errno
import json
import logging
import os
from collections import OrderedDict
from galaxy import util
@@ -19,8 +18,8 @@ DEFAULT_VALUE_TRANSLATION_TYPE = 'template'
class DataManagers:
def __init__(self, app, xml_filename=None):
self.app = app
self.data_managers = OrderedDict()
self.managed_data_tables = OrderedDict()
self.data_managers = {}
self.managed_data_tables = {}
self.tool_path = None
self._reload_count = 0
self.filename = xml_filename or self.app.config.data_manager_config_file
@@ -123,7 +122,7 @@ class DataManager:
self.version = self.DEFAULT_VERSION
self.guid = None
self.tool = None
self.data_tables = OrderedDict()
self.data_tables = {}
self.output_ref_by_data_table = {}
self.move_by_data_table_column = {}
self.value_translation_by_data_table_column = {}
@@ -171,7 +170,7 @@ class DataManager:
data_table_name = data_table_elem.get("name")
assert data_table_name is not None, "A name is required for a data table entry"
if data_table_name not in self.data_tables:
self.data_tables[data_table_name] = OrderedDict()
self.data_tables[data_table_name] = {}
output_elem = data_table_elem.find('output')
if output_elem is not None:
for column_elem in output_elem.findall('column'):
lib/galaxy/tools/execute.py
+1 -1
View File
@@ -152,7 +152,7 @@ class ExecutionTracker:
self.output_datasets = []
self.output_collections = []
self.implicit_collections = collections.OrderedDict()
self.implicit_collections = {}
@property
def param_combinations(self):
lib/galaxy/tools/parameters/__init__.py
+11 -13
View File
@@ -27,7 +27,6 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
If the callback returns a value, it will be replace the old value.
>>> from collections import OrderedDict
>>> from galaxy.util import XML
>>> from galaxy.util.bunch import Bunch
>>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter
@@ -43,9 +42,9 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
>>> i = TextToolParameter(None, XML('<param name="i"/>'))
>>> j = TextToolParameter(None, XML('<param name="j"/>'))
>>> b.name = b.title = 'b'
>>> b.inputs = OrderedDict([ ('c', c), ('d', d) ])
>>> b.inputs = dict([ ('c', c), ('d', d) ])
>>> d.name = d.title = 'd'
>>> d.inputs = OrderedDict([ ('e', e), ('f', f) ])
>>> d.inputs = dict([ ('e', e), ('f', f) ])
>>> f.test_param = g
>>> f.name = 'f'
>>> f.cases = [Bunch(value='true', inputs= {'h': h}), Bunch(value='false', inputs= { 'i': i })]
@@ -54,8 +53,8 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
... print('name=%s, prefix=%s, prefixed_name=%s, prefixed_label=%s, value=%s' % (input.name, prefix, prefixed_name, prefixed_label, value))
... if error:
... print(error)
>>> inputs = OrderedDict([('a', a),('b', b)])
>>> nested = OrderedDict([('a', 1), ('b', [OrderedDict([('c', 3), ('d', [OrderedDict([ ('e', 5), ('f', OrderedDict([ ('g', True), ('h', 7)]))])])])])])
>>> inputs = dict([('a', a),('b', b)])
>>> nested = dict([('a', 1), ('b', [dict([('c', 3), ('d', [dict([ ('e', 5), ('f', dict([ ('g', True), ('h', 7)]))])])])])])
>>> visit_input_values(inputs, nested, visitor)
name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
@@ -103,7 +102,7 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
No value found for 'b 1 > d 1 > j'.
>>> # Other parameters are missing in state
>>> nested = OrderedDict([('b', [OrderedDict([( 'd', [OrderedDict([('f', OrderedDict([('g', True), ('h', 7)]))])])])])])
>>> nested = dict([('b', [dict([( 'd', [dict([('f', dict([('g', True), ('h', 7)]))])])])])])
>>> visit_input_values(inputs, nested, visitor)
name=a, prefix=, prefixed_name=a, prefixed_label=a, value=None
No value found for 'a'.
@@ -282,7 +281,6 @@ def update_dataset_ids(input_values, translate_values, src):
def populate_state(request_context, inputs, incoming, state, errors=None, context=None, check=True, simple_errors=True, input_format='legacy'):
"""
Populates nested state dict from incoming parameter values.
>>> from collections import OrderedDict
>>> from galaxy.util import XML
>>> from galaxy.util.bunch import Bunch
>>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter
@@ -302,15 +300,15 @@ def populate_state(request_context, inputs, incoming, state, errors=None, contex
>>> h = TextToolParameter(None, XML('<param name="h"/>'))
>>> i = TextToolParameter(None, XML('<param name="i"/>'))
>>> b.name = 'b'
>>> b.inputs = OrderedDict([('c', c), ('d', d)])
>>> b.inputs = dict([('c', c), ('d', d)])
>>> d.name = 'd'
>>> d.inputs = OrderedDict([('e', e), ('f', f)])
>>> d.inputs = dict([('e', e), ('f', f)])
>>> f.test_param = g
>>> f.name = 'f'
>>> f.cases = [Bunch(value='true', inputs= { 'h': h }), Bunch(value='false', inputs= { 'i': i })]
>>> inputs = OrderedDict([('a',a),('b',b)])
>>> flat = OrderedDict([('a', 1), ('b_0|c', 2), ('b_0|d_0|e', 3), ('b_0|d_0|f|h', 4), ('b_0|d_0|f|g', True)])
>>> state = OrderedDict()
>>> inputs = dict([('a',a),('b',b)])
>>> flat = dict([('a', 1), ('b_0|c', 2), ('b_0|d_0|e', 3), ('b_0|d_0|f|h', 4), ('b_0|d_0|f|g', True)])
>>> state = {}
>>> populate_state(trans, inputs, flat, state, check=False)
>>> print(state['a'])
1
@@ -322,7 +320,7 @@ def populate_state(request_context, inputs, incoming, state, errors=None, contex
4
>>> # now test with input_format='21.01'
>>> nested = {'a': 1, 'b': [{'c': 2, 'd': [{'e': 3, 'f': {'h': 4, 'g': True}}]}]}
>>> state_new = OrderedDict()
>>> state_new = {}
>>> populate_state(trans, inputs, nested, state_new, check=False, input_format='21.01')
>>> print(state_new['a'])
1
lib/galaxy/tools/parameters/meta.py
+2 -2
View File
@@ -1,7 +1,7 @@
import copy
import itertools
import logging
from collections import namedtuple, OrderedDict
from collections import namedtuple
from galaxy import (
exceptions,
@@ -178,7 +178,7 @@ def expand_meta_parameters(trans, tool, incoming):
if not incoming_key.startswith('__'):
process_key(incoming_key, incoming_value=incoming_value, d=nested_dict)
reordered_incoming = OrderedDict()
reordered_incoming = {}
def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):
if prefixed_name in incoming_copy:
lib/galaxy/tools/toolbox/base.py
+2 -5
View File
@@ -4,10 +4,7 @@ import os
import string
import time
import urllib.request
from collections import (
namedtuple,
OrderedDict
)
from collections import namedtuple
from errno import ENOENT
from urllib.parse import urlparse
@@ -103,7 +100,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
# In-memory dictionary that defines the layout of the tool panel.
self._tool_panel = ToolPanelElements()
self._index = 0
self.data_manager_tools = OrderedDict()
self.data_manager_tools = {}
self._lineage_map = LineageMap(app)
# Sets self._integrated_tool_panel and self._integrated_tool_panel_config_has_contents
self._init_integrated_tool_panel(app.config)
lib/galaxy/tools/wrappers.py
+1 -2
View File
@@ -1,7 +1,6 @@
import logging
import shlex
import tempfile
from collections import OrderedDict
from functools import total_ordering
from galaxy import exceptions
@@ -453,7 +452,7 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
self.collection = collection
elements = collection.elements
element_instances = OrderedDict()
element_instances = {}
element_instance_list = []
for dataset_collection_element in elements:
lib/galaxy/util/__init__.py
-13
View File
@@ -1086,19 +1086,6 @@ def strip_control_characters(s):
return "".join(c for c in unicodify(s) if unicodedata.category(c) != "Cc")
def strip_control_characters_nested(item):
"""Recursively strips control characters from lists, dicts, tuples."""
def visit(path, key, value):
if isinstance(key, str):
key = strip_control_characters(key)
if isinstance(value, str):
value = strip_control_characters(value)
return key, value
return remap(item, visit)
def object_to_string(obj):
return binascii.hexlify(obj)
lib/galaxy/util/permutations.py
+3 -4
View File
@@ -6,7 +6,6 @@ first.
Maybe this doesn't make sense and maybe much of this stuff could be replaced
with itertools product and permutations. These are open questions.
"""
from collections import OrderedDict
from galaxy.exceptions import MessageException
from galaxy.util.bunch import Bunch
@@ -42,9 +41,9 @@ def expand_multi_inputs(inputs, classifier, key_filter=None):
def __split_inputs(inputs, classifier, key_filter):
key_filter = key_filter or (lambda x: True)
single_inputs = OrderedDict()
matched_multi_inputs = OrderedDict()
multiplied_multi_inputs = OrderedDict()
single_inputs = {}
matched_multi_inputs = {}
multiplied_multi_inputs = {}
for input_key in filter(key_filter, inputs):
input_type, expanded_val = classifier(input_key)
lib/galaxy/util/rules_dsl.py
+1 -3
View File
@@ -6,8 +6,6 @@ from typing import List, Type
import yaml
from pkg_resources import resource_stream
from galaxy.util import strip_control_characters_nested
def get_rules_specification():
return yaml.safe_load(resource_stream(__name__, 'rules_dsl_spec.yml'))
@@ -498,7 +496,7 @@ def flat_map(f, items):
class RuleSet:
def __init__(self, rule_set_as_dict):
self.raw_rules = strip_control_characters_nested(rule_set_as_dict["rules"])
self.raw_rules = rule_set_as_dict["rules"]
self.raw_mapping = rule_set_as_dict.get("mapping", [])
@property
lib/galaxy/util/simplegraph.py
+1 -3
View File
@@ -3,8 +3,6 @@ Fencepost-simple graph structure implementation.
"""
# Currently (2013.7.12) only used in easing the parsing of graph datatype data.
from collections import OrderedDict
class SimpleGraphNode:
"""
@@ -58,7 +56,7 @@ class SimpleGraph:
def __init__(self, nodes=None, edges=None):
# use an odict so that edge indeces actually match the final node list indeces
self.nodes = nodes or OrderedDict()
self.nodes = nodes or {}
self.edges = edges or []
def add_node(self, node_id, **data):
lib/galaxy/util/tool_shed/common_util.py
+3 -4
View File
@@ -2,7 +2,6 @@ import errno
import json
import logging
import os
from collections import OrderedDict
from urllib.parse import urljoin
from routes import url_for
@@ -36,16 +35,16 @@ def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_scrip
'migrate', 'scripts',
'%04d_tools.xml' % latest_tool_migration_script_number))
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
migrated_tool_configs_dict = OrderedDict()
migrated_tool_configs_dict = {}
tree, error_message = xml_util.parse_xml(tools_xml_file_path)
if tree is None:
return False, OrderedDict()
return False, {}
root = tree.getroot()
tool_shed = root.get('name')
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed)
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
missing_tool_configs_dict = OrderedDict()
missing_tool_configs_dict = {}
if tool_shed_url:
for elem in root:
if elem.tag == 'repository':
lib/galaxy/util/topsort.py
+4 -5
View File
@@ -36,7 +36,6 @@ then CycleError is raised, and the exception object supports
many methods to help analyze and break the cycles. This requires
a good deal more code than topsort itself!
"""
from collections import OrderedDict
class CycleError(Exception):
@@ -88,7 +87,7 @@ class CycleError(Exception):
def get_preds(self):
if self.preds is not None:
return self.preds
self.preds = preds = OrderedDict()
self.preds = preds = {}
remaining_elts = self.get_elements()
for x in remaining_elts:
preds[x] = []
@@ -117,7 +116,7 @@ class CycleError(Exception):
from random import choice
x = choice(remaining_elts)
answer = []
index = OrderedDict()
index = {}
in_answer = index.has_key
while not in_answer(x):
index[x] = len(answer) # index of x in answer
@@ -130,8 +129,8 @@ class CycleError(Exception):
def _numpreds_and_successors_from_pairlist(pairlist):
numpreds = OrderedDict() # elt -> # of predecessors
successors = OrderedDict() # elt -> list of successors
numpreds = {} # elt -> # of predecessors
successors = {} # elt -> list of successors
for first, second in pairlist:
# make sure every elt is a key in numpreds
if first not in numpreds:
lib/galaxy/util/yaml_util.py
+2 -2
View File
@@ -29,7 +29,7 @@ class OrderedLoader(SafeLoader):
def ordered_load(stream, merge_duplicate_keys=False):
"""
Parse the first YAML document in a stream and produce the corresponding
Python object, using OrderedDicts instead of dicts.
Python object.
If merge_duplicate_keys is True, merge the values of duplicate mapping keys
into a list, as the uWSGI "dumb" YAML parser would do.
@@ -38,7 +38,7 @@ def ordered_load(stream, merge_duplicate_keys=False):
"""
def construct_mapping(loader, node, deep=False):
loader.flatten_mapping(node)
mapping = OrderedDict()
mapping = {}
merged_duplicate = {}
for key_node, value_node in node.value:
key = loader.construct_object(key_node, deep=deep)
lib/galaxy/visualization/plugins/registry.py
+1 -4
View File
@@ -7,7 +7,6 @@ Lower level of visualization framework which does three main things:
import logging
import os
import weakref
from collections import OrderedDict
from galaxy.exceptions import ObjectNotFound
from galaxy.util import (
@@ -66,7 +65,7 @@ class VisualizationsRegistry:
self.additional_template_paths = []
self.directories = []
self.skip_bad_plugins = skip_bad_plugins
self.plugins = OrderedDict()
self.plugins = {}
self.directories = config_directories_from_setting(directories_setting, app.config.root)
self._load_configuration()
self._load_plugins()
@@ -107,8 +106,6 @@ class VisualizationsRegistry:
"""
Search ``self.directories`` for potential plugins, load them, and cache
in ``self.plugins``.
:rtype: OrderedDict
:returns: ``self.plugins``
"""
for plugin_path in self._find_plugins():
try:
lib/galaxy/web/framework/helpers/grids.py
+1 -2
View File
@@ -1,6 +1,5 @@
import logging
import math
from collections import OrderedDict
from json import dumps, loads
from typing import Dict, List, Optional
@@ -492,7 +491,7 @@ class SharingStatusColumn(GridColumn):
def get_accepted_filters(self):
""" Returns a list of accepted filters for this column. """
accepted_filter_labels_and_vals = OrderedDict()
accepted_filter_labels_and_vals = {}
accepted_filter_labels_and_vals["private"] = "private"
accepted_filter_labels_and_vals["shared"] = "shared"
accepted_filter_labels_and_vals["accessible"] = "accessible"
lib/galaxy/web/legacy_framework/grids.py
+1 -2
View File
@@ -1,6 +1,5 @@
import logging
import math
from collections import OrderedDict
from json import dumps, loads
from typing import Dict, List, Optional
@@ -457,7 +456,7 @@ class SharingStatusColumn(GridColumn):
def get_accepted_filters(self):
""" Returns a list of accepted filters for this column. """
accepted_filter_labels_and_vals = OrderedDict()
accepted_filter_labels_and_vals = {}
accepted_filter_labels_and_vals["private"] = "private"
accepted_filter_labels_and_vals["shared"] = "shared"
accepted_filter_labels_and_vals["accessible"] = "accessible"
lib/galaxy/webapps/base/controller.py
+4 -2
View File
@@ -1413,8 +1413,10 @@ class SharableMixin:
skipped = False
class_name = self.manager.model_class.__name__
item = self.get_object(trans, id, class_name, check_ownership=True, check_accessible=True, deleted=False)
if payload and payload.get("action"):
action = payload.get("action")
actions = []
if payload:
actions += payload.get("action").split("-")
for action in actions:
if action == "make_accessible_via_link":
self._make_item_accessible(trans.sa_session, item)
if hasattr(item, "has_possible_members") and item.has_possible_members:
lib/galaxy/webapps/galaxy/api/users.py
+3 -4
View File
@@ -5,7 +5,6 @@ import copy
import json
import logging
import re
from collections import OrderedDict
from markupsafe import escape
from sqlalchemy import (
@@ -713,9 +712,9 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, BaseUIController, Uses
inputs.append({'type': 'section', 'title': filter_title, 'name': filter_type, 'expanded': True, 'inputs': filter_inputs})
def _get_filter_types(self, trans):
return OrderedDict([('toolbox_tool_filters', {'title': 'Tools', 'config': trans.app.config.user_tool_filters}),
('toolbox_section_filters', {'title': 'Sections', 'config': trans.app.config.user_tool_section_filters}),
('toolbox_label_filters', {'title': 'Labels', 'config': trans.app.config.user_tool_label_filters})])
return {'toolbox_tool_filters': {'title': 'Tools', 'config': trans.app.config.user_tool_filters},
'toolbox_section_filters': {'title': 'Sections', 'config': trans.app.config.user_tool_section_filters},
'toolbox_label_filters': {'title': 'Labels', 'config': trans.app.config.user_tool_label_filters}}
@expose_api
def api_key(self, trans, id, payload=None, **kwd):
lib/galaxy/webapps/galaxy/config_schema.yml
+1 -1
View File
@@ -2817,7 +2817,7 @@ mapping:
required: false
default: directory
desc: |
Determines how metadata will be set. Valid values are `directory`, `extended` and `legacy`.
Determines how metadata will be set. Valid values are `directory` and `extended`.
In extended mode jobs will decide if a tool run failed, the object stores
configuration is serialized and made available to the job and is used for
writing output datasets to the object store as part of the job and dynamic
lib/galaxy/webapps/galaxy/controllers/admin.py
+1 -2
View File
@@ -1,7 +1,6 @@
import imp
import logging
import os
from collections import OrderedDict
from datetime import datetime, timedelta
from sqlalchemy import and_, false, or_
@@ -882,7 +881,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
def review_tool_migration_stages(self, trans, **kwd):
message = escape(util.restore_text(kwd.get('message', '')))
status = util.restore_text(kwd.get('status', 'done'))
migration_stages_dict = OrderedDict()
migration_stages_dict = {}
# FIXME: this isn't valid in an installed context
migration_scripts_dir = os.path.abspath(os.path.join(trans.app.config.root, 'lib', 'galaxy', 'tool_shed', 'galaxy_install', 'migrate', 'versions'))
modules = os.listdir(migration_scripts_dir)
lib/galaxy/webapps/galaxy/controllers/history.py
+2 -3
View File
@@ -1,5 +1,4 @@
import logging
from collections import OrderedDict
from markupsafe import escape
from sqlalchemy import (
@@ -485,7 +484,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
items = []
# First go through and group hdas by job, if there is no job they get
# added directly to items
jobs = OrderedDict()
jobs = {}
for hda in history.active_datasets:
if hda.visible is False:
continue
@@ -509,7 +508,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
else:
jobs[job] = [(hda, None)]
# Second, go through the jobs and connect to workflows
wf_invocations = OrderedDict()
wf_invocations = {}
for job, hdas in jobs.items():
# Job is attached to a workflow step, follow it to the
# workflow_invocation and group
lib/galaxy/webapps/reports/controllers/history.py
+2 -4
View File
@@ -1,4 +1,3 @@
import collections
import logging
import sqlalchemy as sa
@@ -111,7 +110,7 @@ class History(BaseUIController):
users = users[:user_cutoff]
# to keep ordered
data = collections.OrderedDict()
data = {}
for user in users:
dataset = datasets.get(user, [0, 0])
history = histories.get(user, 0)
@@ -172,8 +171,7 @@ class History(BaseUIController):
possible_status = {"ok": 0, "upload": 1, "paused": 2, "queued": 3, "error": 4, "discarded": 5}
number_of_possible_status = len(possible_status) + 1 # + 1 to handle unknown status!
# to keep ordered
datas = collections.OrderedDict()
datas = {}
for no, name in enumerate(names):
if name not in datas:
if user_cutoff > 0:
lib/galaxy/webapps/reports/controllers/tools.py
+5 -6
View File
@@ -1,4 +1,3 @@
import collections
import logging
from datetime import timedelta
@@ -85,7 +84,7 @@ class Tools(BaseUIController):
lambda v: tools_and_jobs_ok.get(v, 0),
lambda v: tools_and_jobs_error.get(v, 0))
data = collections.OrderedDict()
data = {}
# select count(id), tool_id from job where state='ok' group by tool_id;
tools_and_jobs_ok = sa.select((galaxy.model.Job.table.c.tool_id .label('tool'),
@@ -139,7 +138,7 @@ class Tools(BaseUIController):
if tool is None:
raise TypeError("Tool can't be None")
data = collections.OrderedDict()
data = {}
# select count(id), create_time from job where state='ok' and tool_id=$tool group by date;
date_and_jobs_ok = sa.select((sa.func.date(galaxy.model.Job.table.c.create_time).label('date'),
@@ -194,7 +193,7 @@ class Tools(BaseUIController):
color = True if kwd.get("color", '') == "True" else False
data = {}
ordered_data = collections.OrderedDict()
ordered_data = {}
sort_keys = (
lambda v: v.lower(),
@@ -260,7 +259,7 @@ class Tools(BaseUIController):
if tool is None:
raise ValueError("Tool can't be None")
ordered_data = collections.OrderedDict()
ordered_data = {}
sort_keys = [(lambda v, i=i: v[i]) for i in range(4)]
jobs_times = sa.select((sa.func.date_trunc('month', galaxy.model.Job.table.c.create_time).label('date'),
@@ -314,7 +313,7 @@ class Tools(BaseUIController):
else:
counter[error[0]] = [1, error[1]]
data = collections.OrderedDict()
data = {}
keys = list(counter.keys())
if cutoff:
keys = keys[:cutoff]
lib/galaxy/workflow/extract.py
+1 -2
View File
@@ -2,7 +2,6 @@
histories.
"""
import logging
from collections import OrderedDict
from galaxy import exceptions, model
from galaxy.tool_util.parser import ToolOutputCollectionPart
@@ -196,7 +195,7 @@ class WorkflowSummary:
history = trans.get_history()
self.history = history
self.warnings = set()
self.jobs = OrderedDict()
self.jobs = {}
self.job_id2representative_job = {} # map a non-fake job id to its representative job
self.implicit_map_jobs = []
self.collection_types = {}
lib/galaxy/workflow/modules.py
+17 -19
View File
@@ -4,7 +4,7 @@ Modules used in building workflows
import json
import logging
import re
from collections import defaultdict, OrderedDict
from collections import defaultdict
import packaging.version
@@ -707,7 +707,7 @@ class InputDataModule(InputModule):
def get_inputs(self):
parameter_def = self._parse_state_into_dict()
optional = parameter_def["optional"]
inputs = OrderedDict()
inputs = {}
inputs["optional"] = optional_param(optional)
inputs["format"] = format_param(self.trans, parameter_def.get("format"))
return inputs
@@ -730,7 +730,7 @@ class InputDataCollectionModule(InputModule):
{"value": "list:paired", "label": "List of Dataset Pairs"},
]
input_collection_type = TextToolParameter(None, collection_type_source)
inputs = OrderedDict()
inputs = {}
inputs["collection_type"] = input_collection_type
inputs["optional"] = optional_param(optional)
inputs["format"] = format_param(self.trans, parameter_def.get("format"))
@@ -859,7 +859,7 @@ class InputParameterModule(WorkflowModule):
when_this_type = ConditionalWhen()
when_this_type.value = param_type
when_this_type.inputs = OrderedDict()
when_this_type.inputs = {}
when_this_type.inputs["optional"] = optional_cond
specify_default_checked = "default" in parameter_def
@@ -871,24 +871,24 @@ class InputParameterModule(WorkflowModule):
when_specify_default_true = ConditionalWhen()
when_specify_default_true.value = "true"
when_specify_default_true.inputs = OrderedDict()
when_specify_default_true.inputs = {}
when_specify_default_true.inputs["default"] = input_default_value
when_specify_default_false = ConditionalWhen()
when_specify_default_false.value = "false"
when_specify_default_false.inputs = OrderedDict()
when_specify_default_false.inputs = {}
specify_default_cond_cases = [when_specify_default_true, when_specify_default_false]
specify_default_cond.cases = specify_default_cond_cases
when_true = ConditionalWhen()
when_true.value = "true"
when_true.inputs = OrderedDict()
when_true.inputs = {}
when_true.inputs["default"] = specify_default_cond
when_false = ConditionalWhen()
when_false.value = "false"
when_false.inputs = OrderedDict()
when_false.inputs = {}
optional_cases = [when_true, when_false]
optional_cond.cases = optional_cases
@@ -916,19 +916,19 @@ class InputParameterModule(WorkflowModule):
when_restrict_none = ConditionalWhen()
when_restrict_none.value = "none"
when_restrict_none.inputs = OrderedDict()
when_restrict_none.inputs = {}
when_restrict_connections = ConditionalWhen()
when_restrict_connections.value = "onConnections"
when_restrict_connections.inputs = OrderedDict()
when_restrict_connections.inputs = {}
when_restrict_static_restrictions = ConditionalWhen()
when_restrict_static_restrictions.value = "staticRestrictions"
when_restrict_static_restrictions.inputs = OrderedDict()
when_restrict_static_restrictions.inputs = {}
when_restrict_static_suggestions = ConditionalWhen()
when_restrict_static_suggestions.value = "staticSuggestions"
when_restrict_static_suggestions.inputs = OrderedDict()
when_restrict_static_suggestions.inputs = {}
# Repeats don't work - so use common separated list for now.
@@ -953,7 +953,7 @@ class InputParameterModule(WorkflowModule):
cases.append(when_this_type)
parameter_type_cond.cases = cases
return OrderedDict([("parameter_definition", parameter_type_cond)])
return {"parameter_definition": parameter_type_cond}
def get_runtime_inputs(self, connections=None, **kwds):
parameter_def = self._parse_state_into_dict()
@@ -1445,12 +1445,10 @@ class ToolModule(WorkflowModule):
if not collection_type and tool_output.structure.collection_type_from_rules:
rule_param = tool_output.structure.collection_type_from_rules
if rule_param in self.state.inputs:
rule_json_str = self.state.inputs[rule_param]
if rule_json_str: # initialized to None...
rules = rule_json_str
if rules:
rule_set = RuleSet(rules)
collection_type = rule_set.collection_type
rules = self.state.inputs[rule_param]
if rules:
rule_set = RuleSet(rules)
collection_type = rule_set.collection_type
extra_kwds["collection_type"] = collection_type
extra_kwds["collection_type_source"] = tool_output.structure.collection_type_source
formats = ['input'] # TODO: fix
lib/galaxy/workflow/run.py
+1 -2
View File
@@ -1,6 +1,5 @@
import logging
import uuid
from collections import OrderedDict
from galaxy import model
from galaxy.util import ExecutionTimer
@@ -274,7 +273,7 @@ STEP_OUTPUT_DELAYED = object()
class WorkflowProgress:
def __init__(self, workflow_invocation, inputs_by_step_id, module_injector, param_map, jobs_per_scheduling_iteration=-1):
self.outputs = OrderedDict()
self.outputs = {}
self.module_injector = module_injector
self.workflow_invocation = workflow_invocation
self.inputs_by_step_id = inputs_by_step_id
lib/tool_shed/dependencies/attribute_handlers.py
+3 -4
View File
@@ -1,6 +1,5 @@
import copy
import logging
from collections import OrderedDict
from galaxy.util import asbool
from galaxy.web import url_for
@@ -71,8 +70,8 @@ class RepositoryDependencyAttributeHandler:
if len(sub_elems) > 0:
# At this point, a <repository> tag will point only to a package.
# <package name="xorg_macros" version="1.17.1" />
# Coerce the list to an OrderedDict().
sub_elements = OrderedDict()
# Coerce the list to dict.
sub_elements = {}
packages = []
for sub_elem in sub_elems:
sub_elem_type = sub_elem.tag
@@ -88,7 +87,7 @@ class RepositoryDependencyAttributeHandler:
# We're exporting the repository, so eliminate all toolshed and changeset_revision attributes
# from the <repository> tag.
if toolshed or changeset_revision:
attributes = OrderedDict()
attributes = {}
attributes['name'] = name
attributes['owner'] = owner
prior_installation_required = elem.get('prior_installation_required')
lib/tool_shed/grids/repository_grids.py
+1 -2
View File
@@ -1,4 +1,3 @@
import json
import logging
from markupsafe import escape as escape_html
@@ -181,7 +180,7 @@ class RepositoryGrid(grids.Grid):
class EmailAlertsColumn(grids.TextColumn):
def get_value(self, trans, grid, repository):
if trans.user and repository.email_alerts and trans.user.email in json.loads(repository.email_alerts):
if trans.user and trans.user.email in repository.email_alerts:
return 'yes'
return ''
lib/tool_shed/repository_types/registry.py
+1 -2
View File
@@ -1,5 +1,4 @@
import logging
from collections import OrderedDict
from . import (
repository_suite_definition,
@@ -13,7 +12,7 @@ log = logging.getLogger(__name__)
class Registry:
def __init__(self):
self.repository_types_by_label = OrderedDict()
self.repository_types_by_label = {}
self.repository_types_by_label['unrestricted'] = unrestricted.Unrestricted()
self.repository_types_by_label['repository_suite_definition'] = repository_suite_definition.RepositorySuiteDefinition()
self.repository_types_by_label['tool_dependency_definition'] = tool_dependency_definition.ToolDependencyDefinition()
lib/tool_shed/util/commit_util.py
+1 -3
View File
@@ -1,6 +1,5 @@
import bz2
import gzip
import json
import logging
import os
import shutil
@@ -72,8 +71,7 @@ def check_file_contents_for_email_alerts(app):
admin_users = app.config.get("admin_users", "").split(",")
for repository in sa_session.query(app.model.Repository) \
.filter(app.model.Repository.table.c.email_alerts != null()):
email_alerts = json.loads(repository.email_alerts)
for user_email in email_alerts:
for user_email in repository.email_alerts:
if user_email in admin_users:
return True
return False
lib/tool_shed/util/review_util.py
+1 -2
View File
@@ -1,5 +1,4 @@
import logging
from collections import OrderedDict
from sqlalchemy import and_
@@ -74,7 +73,7 @@ def get_previous_repository_reviews(app, repository, changeset_revision):
"""
repo = repository.hg_repo
reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews]
previous_reviews_dict = OrderedDict()
previous_reviews_dict = {}
for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision):
previous_changeset_revision = str(repo[changeset])
if previous_changeset_revision in reviewed_revision_hashes:
lib/tool_shed/util/shed_util_common.py
+1 -1
View File
@@ -385,7 +385,7 @@ def handle_email_alerts(app, host, repository, content_alert_str='', new_repo_al
email_alerts.append(user.email)
else:
subject = "Galaxy tool shed update alert for repository named %s" % str(repository.name)
email_alerts = json.loads(repository.email_alerts)
email_alerts = repository.email_alerts
for email in email_alerts:
to = email.strip()
# Send it
lib/tool_shed/webapp/controllers/repository.py
+7 -23
View File
@@ -1658,10 +1658,6 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
alerts = kwd.get('alerts', '')
alerts_checked = CheckboxField.is_checked(alerts)
category_ids = util.listify(kwd.get('category_id', ''))
if repository.email_alerts:
email_alerts = json.loads(repository.email_alerts)
else:
email_alerts = []
allow_push = kwd.get('allow_push', '')
error = False
user = trans.user
@@ -1714,14 +1710,12 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
elif kwd.get('receive_email_alerts_button', False):
flush_needed = False
if alerts_checked:
if user.email not in email_alerts:
email_alerts.append(user.email)
repository.email_alerts = json.dumps(email_alerts)
if user.email not in repository.email_alerts:
repository.email_alerts.append(user.email)
flush_needed = True
else:
if user.email in email_alerts:
email_alerts.remove(user.email)
repository.email_alerts = json.dumps(email_alerts)
if user.email in repository.email_alerts:
repository.email_alerts.remove(user.email)
flush_needed = True
if flush_needed:
trans.sa_session.add(repository)
@@ -1743,7 +1737,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
for obj in options:
label = obj.username
allow_push_select_field.add_option(label, trans.security.encode_id(obj.id))
checked = alerts_checked or user.email in email_alerts
checked = alerts_checked or user.email in repository.email_alerts
alerts_check_box = CheckboxField('alerts', value=checked)
changeset_revision_select_field = grids_util.build_changeset_revision_select_field(trans,
repository,
@@ -2273,19 +2267,14 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
flush_needed = False
for repository_id in repository_ids:
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
if repository.email_alerts:
email_alerts = json.loads(repository.email_alerts)
else:
email_alerts = []
email_alerts = repository.email_alerts
if user.email in email_alerts:
email_alerts.remove(user.email)
repository.email_alerts = json.dumps(email_alerts)
trans.sa_session.add(repository)
flush_needed = True
total_alerts_removed += 1
else:
email_alerts.append(user.email)
repository.email_alerts = json.dumps(email_alerts)
trans.sa_session.add(repository)
flush_needed = True
total_alerts_added += 1
@@ -2568,10 +2557,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
display_reviews = kwd.get('display_reviews', False)
alerts = kwd.get('alerts', '')
alerts_checked = CheckboxField.is_checked(alerts)
if repository.email_alerts:
email_alerts = json.loads(repository.email_alerts)
else:
email_alerts = []
email_alerts = repository.email_alerts
repository_dependencies = None
user = trans.user
if user and kwd.get('receive_email_alerts_button', False):
@@ -2579,12 +2565,10 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
if alerts_checked:
if user.email not in email_alerts:
email_alerts.append(user.email)
repository.email_alerts = json.dumps(email_alerts)
flush_needed = True
else:
if user.email in email_alerts:
email_alerts.remove(user.email)
repository.email_alerts = json.dumps(email_alerts)
flush_needed = True
if flush_needed:
trans.sa_session.add(repository)
lib/tool_shed/webapp/controllers/repository_review.py
+2 -3
View File
@@ -1,5 +1,4 @@
import logging
from collections import OrderedDict
from sqlalchemy import (
and_,
@@ -240,7 +239,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings):
status = kwd.get('status', 'done')
review_id = kwd.get('id', None)
review = review_util.get_review(trans.app, review_id)
components_dict = OrderedDict()
components_dict = {}
for component in review_util.get_components(trans.app):
components_dict[component.name] = dict(component=component, component_review=None)
repository = review.repository
@@ -487,7 +486,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings):
repo = repository.hg_repo
metadata_revision_hashes = [metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions]
reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews]
reviews_dict = OrderedDict()
reviews_dict = {}
for changeset in hg_util.get_reversed_changelog_changesets(repo):
changeset_revision = str(repo[changeset])
if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes:
lib/tool_shed/webapp/model/mapping.py
+1 -1
View File
@@ -118,7 +118,7 @@ Repository.table = Table("repository", metadata,
Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
Column("private", Boolean, default=False),
Column("deleted", Boolean, index=True, default=False),
Column("email_alerts", JSONType, nullable=True),
Column("email_alerts", JSONType, nullable=True, default=list),
Column("times_downloaded", Integer),
Column("deprecated", Boolean, default=False))
packages/data/requirements.txt
+1
View File
@@ -13,5 +13,6 @@ pysam
social-auth-core[openidconnect]==3.3.0
SQLAlchemy
sqlalchemy-migrate
sqlalchemy-mutable
sqlalchemy-utils
WebOb
pyproject.toml
+1
View File
@@ -64,6 +64,7 @@ social-auth-core = {version = "==3.3.0", extras = ["openidconnect"]}
sortedcontainers = "*"
SQLAlchemy = "*"
sqlalchemy-migrate = "*"
sqlalchemy-mutable = "*"
SQLAlchemy-Utils = "!=0.36.8" # https://github.com/kvesteri/sqlalchemy-utils/issues/462
sqlitedict = "*"
sqlparse = "*"
test/unit/objectstore/test_objectstore.py
+9
View File
@@ -273,6 +273,15 @@ def test_hierarchical_store():
_assert_key_has_value(as_dict, "type", "hierarchical")
def test_concrete_name_without_objectstore_id():
for config_str in [HIERARCHICAL_TEST_CONFIG, HIERARCHICAL_TEST_CONFIG_YAML]:
with TestConfig(config_str) as (directory, object_store):
files1_desc = object_store.get_concrete_store_description_markdown(MockDataset(3))
files1_name = object_store.get_concrete_store_name(MockDataset(3))
assert files1_desc is None
assert files1_name is None
MIXED_STORE_BY_HIERARCHICAL_TEST_CONFIG = """<?xml version="1.0"?>
<object_store type="hierarchical">
<backends>
test/unit/tools/test_metadata.py
-12
View File
@@ -33,10 +33,6 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
super().tearDown()
self.metadata_compute_strategy = None
def test_simple_output_legacy(self):
self.app.config.metadata_strategy = "legacy"
self._test_simple_output()
def test_simple_output_directory(self):
self.app.config.metadata_strategy = "directory"
self._test_simple_output()
@@ -66,10 +62,6 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
assert output_dataset.metadata.data_lines == 2
assert output_dataset.metadata.sequences == 1
def test_primary_dataset_output_extension_legacy(self):
self.app.config.metadata_strategy = "legacy"
self._test_primary_dataset_output_extension()
def test_primary_dataset_output_extension_directory(self):
self.app.config.metadata_strategy = "directory"
self._test_primary_dataset_output_extension()
@@ -99,10 +91,6 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
assert output_dataset.metadata.data_lines == 2
assert output_dataset.metadata.sequences == 1
def test_primary_dataset_output_metadata_override_legacy(self):
self.app.config.metadata_strategy = "legacy"
self._test_primary_dataset_output_metadata_override()
def test_primary_dataset_output_metadata_override_directory(self):
self.app.config.metadata_strategy = "directory"
self._test_primary_dataset_output_metadata_override()
test/unit/util/test_utils.py
-11
View File
@@ -21,17 +21,6 @@ def test_strip_control_characters():
assert util.strip_control_characters(s) == 'bla'
def test_strip_control_characters_nested():
s = '\x00bla'
stripped_s = 'bla'
list_ = [s]
t = (s, 'blub')
d = {42: s}
assert util.strip_control_characters_nested(list_)[0] == stripped_s
assert util.strip_control_characters_nested(t)[0] == stripped_s
assert util.strip_control_characters_nested(d)[42] == stripped_s
def test_parse_xml_string():
section = util.parse_xml_string(SECTION_XML)
_verify_section(section)