From 08cde1aca7cefa97ffc0d2451075a2575b2526fc Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 15 Sep 2020 16:44:18 +0200 Subject: [PATCH 1/4] Decrease pysam verbosity Can be reverted once https://github.com/pysam-developers/pysam/issues/939 is fixed --- lib/galaxy/datatypes/binary.py | 5 +++-- lib/galaxy/visualization/data_providers/genome.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 6ede4c2614f..58ff4e02827 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -33,6 +33,7 @@ from galaxy.util.checkers import is_bz2, is_gzip from . import data, dataproviders log = logging.getLogger(__name__) +pysam.set_verbosity(0) # Currently these supported binary data types must be manually set on upload @@ -489,9 +490,9 @@ class Bam(BamNative): # we start another process and discard stderr. if index_flag == '-b': # IOError: No such file or directory: '-b' if index_flag is set to -b (pysam 0.15.4) - cmd = ['python', '-c', "import pysam; pysam.index('%s', '%s')" % (file_name, index_name)] + cmd = ['python', '-c', "import pysam; pysam.set_verbosity(0); pysam.index('{}', '{}')".format(file_name, index_name)] else: - cmd = ['python', '-c', "import pysam; pysam.index('%s', '%s', '%s')" % (index_flag, file_name, index_name)] + cmd = ['python', '-c', "import pysam; pysam.set_verbosity(0); pysam.index('{}', '{}', '{}')".format(index_flag, file_name, index_name)] with open(os.devnull, 'w') as devnull: subprocess.check_call(cmd, stderr=devnull, shell=False) needs_sorting = False diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 1e06e82bcab..580ec1df22b 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -27,6 +27,7 @@ from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and # Utility functions. # +pysam.set_verbosity(0) PYSAM_INDEX_SYMLINK_NECESSARY = packaging.version.parse(pysam.__version__) <= packaging.version.parse('0.13.0') From f31b0711120e2485b52e9f102f0652119a71dc53 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 15 Sep 2020 17:12:38 +0200 Subject: [PATCH 2/4] Comment on verbository changes --- lib/galaxy/datatypes/binary.py | 2 ++ lib/galaxy/visualization/data_providers/genome.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 58ff4e02827..1fa6bd500a0 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -33,6 +33,8 @@ from galaxy.util.checkers import is_bz2, is_gzip from . import data, dataproviders log = logging.getLogger(__name__) +# pysam 0.16.0.1 emits logs containing the word 'Error', this can confuse the stdout/stderr checkers. +# Can be be removed once https://github.com/pysam-developers/pysam/issues/939 is resolved. pysam.set_verbosity(0) # Currently these supported binary data types must be manually set on upload diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 580ec1df22b..c9c1a82b2c2 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -27,6 +27,8 @@ from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and # Utility functions. # +# pysam 0.16.0.1 emits logs containing the word 'Error', this can confuse the stdout/stderr checkers. +# Can be be removed once https://github.com/pysam-developers/pysam/issues/939 is resolved. pysam.set_verbosity(0) PYSAM_INDEX_SYMLINK_NECESSARY = packaging.version.parse(pysam.__version__) <= packaging.version.parse('0.13.0') From 3fbfa558b35be8e7658f2b2a15156aab236cf7b4 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 17 Sep 2020 11:41:58 +0200 Subject: [PATCH 3/4] Fix change datatype PJA not applied when loaded from server Fixes https://github.com/galaxyproject/galaxy/issues/10228. I think I broke this in https://github.com/galaxyproject/galaxy/pull/9676/ where I added `ChangeOutputDatatype`, which I don't think is a thing, but also I removed the `force_datatype` injection into output parameters, since we can restore this on the client (good), but then I only added the restore logic in `update_field_data`, but not `init_field_data` (bad). You can test this by adding any tool that produces text output (cat1 for instance). Add a change PJA datatype to binary, now you can't connect the output to another cat1 input. Reload the workflow, and you will be able to do it because the editor didn't apply the PJA when creating the workflow nodes. This PR fixes that. --- .../scripts/mvc/workflow/workflow-node.js | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/client/galaxy/scripts/mvc/workflow/workflow-node.js b/client/galaxy/scripts/mvc/workflow/workflow-node.js index 3d09ce6a856..de34ad732ba 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-node.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-node.js @@ -60,11 +60,14 @@ export class Node { } return changed; } - changeOutputDatatype(outputName, datatype) { + setDatatype(outputName, datatype) { const output_terminal = this.output_terminals[outputName]; const output = this.nodeView.outputViews[outputName].output; - output_terminal.force_datatype = datatype; output.force_datatype = datatype; + output_terminal.force_datatype = datatype; + output_terminal.destroyInvalidConnections(); + } + changeOutputDatatype(outputName, datatype) { if (datatype) { this.post_job_actions["ChangeDatatypeAction" + outputName] = { action_arguments: { newtype: datatype }, @@ -74,8 +77,8 @@ export class Node { } else { delete this.post_job_actions["ChangeDatatypeAction" + outputName]; } + this.setDatatype(outputName, datatype); this.markChanged(); - output_terminal.destroyInvalidConnections(); } connectedOutputTerminals() { return this._connectedTerminals(this.output_terminals); @@ -241,6 +244,11 @@ export class Node { } $.each(data.outputs, (i, output) => { nodeView.addDataOutput(output); + const changeOutputDatatype = node.post_job_actions["ChangeDatatypeAction" + output.name]; + node.setDatatype( + output.name, + changeOutputDatatype ? changeOutputDatatype.action_arguments["newtype"] : null + ); }); nodeView.render(); this.app.node_changed(this); @@ -294,20 +302,17 @@ export class Node { } else { // the output already exists, but the output formats may have changed. // Therefore we update the datatypes and destroy invalid connections. - node.output_terminals[output.name].datatypes = output.extensions; - const changeOutputDatatype = node.post_job_actions["ChangeOutputDatatype" + output.name]; - if (changeOutputDatatype) { - node.output_terminals[output.name].force_datatype = - changeOutputDatatype.action_arguments["newtype"]; - } else { - node.output_terminals[output.name].force_datatype = null; - } if (node.type == "parameter_input") { node.output_terminals[output.name].attributes.type = output.type; } node.output_terminals[output.name].optional = output.optional; - node.output_terminals[output.name].destroyInvalidConnections(); + node.output_terminals[output.name].datatypes = output.extensions; } + const changeOutputDatatype = node.post_job_actions["ChangeDatatypeAction" + output.name]; + node.setDatatype( + output.name, + changeOutputDatatype ? changeOutputDatatype.action_arguments["newtype"] : null + ); }); this.tool_state = data.tool_state; this.config_form = data.config_form; From 8ef1a8f51e64afd73ccf745090f584cbd0a1e452 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Sep 2020 13:28:28 +0200 Subject: [PATCH 4/4] Make force_datatype a computated property This is effectively the same as https://github.com/galaxyproject/galaxy/pull/10237 which doesn't apply to dev anymore. --- client/src/components/Workflow/Editor/Node.vue | 2 -- client/src/components/Workflow/Editor/NodeOutput.vue | 2 -- .../components/Workflow/Editor/modules/terminals.js | 12 ++++-------- client/tests/qunit/tests/workflow_editor_tests.js | 3 ++- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/client/src/components/Workflow/Editor/Node.vue b/client/src/components/Workflow/Editor/Node.vue index b2a876eb695..27674b9e88d 100644 --- a/client/src/components/Workflow/Editor/Node.vue +++ b/client/src/components/Workflow/Editor/Node.vue @@ -414,8 +414,6 @@ export default { } const outputName = output.name; const outputTerminal = this.outputTerminals[outputName]; - outputTerminal.force_datatype = datatype; - output.force_datatype = datatype; if (datatype) { this.postJobActions["ChangeDatatypeAction" + outputName] = { action_arguments: { newtype: datatype }, diff --git a/client/src/components/Workflow/Editor/NodeOutput.vue b/client/src/components/Workflow/Editor/NodeOutput.vue index 060b033f2b5..59c6f3a26c5 100644 --- a/client/src/components/Workflow/Editor/NodeOutput.vue +++ b/client/src/components/Workflow/Editor/NodeOutput.vue @@ -74,7 +74,6 @@ export default { collection_type: collection_type, collection_type_source: collection_type_source, datatypes: output.extensions, - force_datatype: output.force_datatype, optional: output.optional, }); } else if (output.parameter) { @@ -91,7 +90,6 @@ export default { name: output.name, element: this.$refs.terminal, datatypes: output.extensions, - force_datatype: output.force_datatype, optional: output.optional, }); } diff --git a/client/src/components/Workflow/Editor/modules/terminals.js b/client/src/components/Workflow/Editor/modules/terminals.js index df022d214e0..9e2fab488d3 100644 --- a/client/src/components/Workflow/Editor/modules/terminals.js +++ b/client/src/components/Workflow/Editor/modules/terminals.js @@ -589,18 +589,14 @@ class BaseOutputTerminal extends Terminal { super(attr); this.datatypes = attr.datatypes; this.optional = attr.optional; - this.force_datatype = attr.force_datatype; + } + get force_datatype() { + const changeOutputDatatype = this.node.postJobActions["ChangeDatatypeAction" + this.name]; + return changeOutputDatatype ? changeOutputDatatype.action_arguments["newtype"] : null; } update(output) { this.datatypes = output.datatypes || output.extensions; this.optional = output.optional; - this.force_datatype = output.force_datatype; - const changeOutputDatatype = this.node.postJobActions["ChangeOutputDatatype" + output.name]; - if (changeOutputDatatype) { - this.node.outputTerminals[output.name].force_datatype = changeOutputDatatype.action_arguments["newtype"]; - } else { - this.node.outputTerminals[output.name].force_datatype = null; - } } } diff --git a/client/tests/qunit/tests/workflow_editor_tests.js b/client/tests/qunit/tests/workflow_editor_tests.js index 040e2982de2..0e50080f2c3 100644 --- a/client/tests/qunit/tests/workflow_editor_tests.js +++ b/client/tests/qunit/tests/workflow_editor_tests.js @@ -35,6 +35,7 @@ const nodeData = { class Node { constructor(attr = {}) { this.element = attr.element; + this.postJobActions = {}; this.inputTerminals = {}; this.outputTerminals = {}; } @@ -326,7 +327,7 @@ QUnit.test("Collection output can connect to same collection input type", functi datatypes: "txt", collection_type: "list", }); - outputTerminal.node = {}; + outputTerminal.node = {postJobActions: {}}; assert.ok( inputTerminal.canAccept(outputTerminal).canAccept, "Input terminal " + inputTerminal + " can not accept " + outputTerminal