From 91c2676946e6893cd25ce021435d3e4a513987a5 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 25 Feb 2020 17:28:31 +0100 Subject: [PATCH 01/38] Add tool recommendations for tool run --- .../galaxy/scripts/mvc/tool-recommendation.js | 197 +++++++++++++++ client/galaxy/scripts/mvc/tool/tool-form.js | 9 + client/galaxy/style/scss/base.scss | 20 ++ .../tool_recommendations_overwrite.yml.sample | 48 ++++ doc/source/admin/galaxy_options.rst | 64 +++++ lib/galaxy/config/sample/galaxy.yml.sample | 29 +++ .../pipfiles/default/pinned-requirements.txt | 2 + lib/galaxy/managers/configuration.py | 5 + lib/galaxy/webapps/galaxy/api/workflows.py | 224 ++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 1 + lib/galaxy/webapps/galaxy/config_schema.yml | 40 ++++ lib/galaxy_test/api/test_workflows.py | 18 ++ 12 files changed, 657 insertions(+) create mode 100644 client/galaxy/scripts/mvc/tool-recommendation.js create mode 100644 config/tool_recommendations_overwrite.yml.sample diff --git a/client/galaxy/scripts/mvc/tool-recommendation.js b/client/galaxy/scripts/mvc/tool-recommendation.js new file mode 100644 index 00000000000..9751d21db29 --- /dev/null +++ b/client/galaxy/scripts/mvc/tool-recommendation.js @@ -0,0 +1,197 @@ +import Backbone from "backbone"; +import Utils from "utils/utils"; +import * as d3 from "d3"; +import { getAppRoot } from "onload/loadConfig"; +import $ from "jquery"; + + +var ToolRecommendationView = Backbone.View.extend({ + el: "#tool-recommendation-view", + + initialize: function(options) { + let toolId = options.toolId || ""; + let self = this; + if (toolId.indexOf("/") > 0) { + let toolIdSlash = toolId.split("/"); + toolId = toolIdSlash[toolIdSlash.length - 2]; + } + Utils.request({ + type: "POST", + url: `${getAppRoot()}api/workflows/get_tool_predictions`, + data: {"tool_sequence": toolId}, + success: data => { + // get datatypes mapping + let datatypes_mapping = JSON.parse( + $.ajax({ + url: `${getAppRoot()}api/datatypes/mapping`, + async: false + }).responseText + ); + let extToType = datatypes_mapping.ext_to_class_name; + let typeToType = datatypes_mapping.class_to_classes; + let predData = data.predicted_data; + if (data !== null && predData.children.length > 0) { + let filteredData = {}; + let compatibleTools = {}; + let filteredChildren = []; + let outputDatatypes = predData["o_extensions"]; + for (const [index, nameObj] of predData.children.entries()) { + let inputDatatypes = nameObj["i_extensions"]; + for (const out_t of outputDatatypes.entries()) { + for(const in_t of inputDatatypes.entries()) { + let child = extToType[out_t[1]]; + let parent = extToType[in_t[1]]; + if (((typeToType[child] && parent in typeToType[child]) === true) || + out_t[1] === "input" || + out_t[1] === "_sniff_" || + out_t[1] === "input_collection") { + compatibleTools[nameObj["tool_id"]] = nameObj["name"]; + break + } + } + } + } + for (let id in compatibleTools) { + for (const [index, nameObj] of predData.children.entries()) { + if (nameObj["tool_id"] === id) { + filteredChildren.push(nameObj); + break + } + } + } + filteredData["o_extensions"] = predData["o_extensions"]; + filteredData["name"] = predData["name"]; + filteredData["children"] = filteredChildren; + if (filteredChildren.length > 0 && predData["is_deprecated"] === false) { + self.$el.append("
You have used " + filteredData.name + " tool. For further analysis, you could try using the following/recommended tools. The recommended tools are shown in the decreasing order of their scores predicted using machine learning analysis on workflows. A tool with a higher score (closer to 100%) may fit better as the following tool than a tool with a lower score. Please click on one of the following/recommended tools to open its definition.
"); + self.render_tree(filteredData); + } + else if(predData["is_deprecated"] === true) { + self.$el.append("
You have used " + predData.name + " tool. " + predData["message"] + ".
"); + } + } + } + }); + }, + + render_tree: function(predicted_data) { + let margin = {top: 20, right: 30, bottom: 20, left: 250}, + width = 900 - margin.right - margin.left, + height = 300 - margin.top - margin.bottom; + let i = 0, + duration = 750, + root; + let tree = d3.layout.tree() + .size([height, width]); + let diagonal = d3.svg.diagonal() + .projection(d => { return [d.y, d.x]; }); + let svg = d3.select("#tool-recommendation-view").append("svg") + .attr("width", width + margin.right + margin.left) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + function update(source) { + // Compute the new tree layout. + let nodes = tree.nodes(root).reverse(), + links = tree.links(nodes); + // Normalize for fixed-depth. + nodes.forEach(d => { d.y = d.depth * 180; }); + // Update the nodes… + let node = svg.selectAll("g.node") + .data(nodes, d => { return d.id || (d.id = ++i); }); + // Enter any new nodes at the parent's previous position. + let nodeEnter = node.enter().append("g") + .attr("class", "node") + .attr("transform", d => { return "translate(" + source.y0 + "," + source.x0 + ")"; }) + .on("click", click); + nodeEnter.append("circle") + .attr("r", 1e-6) + .style("fill", d => { return d._children ? "lightsteelblue" : "#fff"; }); + nodeEnter.append("text") + .attr("x", d => { return d.children || d._children ? -10 : 10; }) + .attr("dy", ".35em") + .attr("text-anchor", d => { return d.children || d._children ? "end" : "start"; }) + .text(d => { return d.name; }) + .style("fill-opacity", 1e-6); + nodeEnter.append("title") + .text(d => { return d.children || d._children ? "Click to collapse" : "Click to open tool definition"; }) + // Transition nodes to their new position. + let nodeUpdate = node.transition() + .duration(duration) + .attr("transform", d => { return "translate(" + d.y + "," + d.x + ")"; }); + nodeUpdate.select("circle") + .attr("r", 4.5) + .style("fill", d => { return d._children ? "lightsteelblue" : "#fff"; }); + nodeUpdate.select("text") + .style("fill-opacity", 1); + // Transition exiting nodes to the parent's new position. + let nodeExit = node.exit().transition() + .duration(duration) + .attr("transform", d => { return "translate(" + source.y + "," + source.x + ")"; }) + .remove(); + nodeExit.select("circle") + .attr("r", 1e-6); + nodeExit.select("text") + .style("fill-opacity", 1e-6); + // Update the links… + let link = svg.selectAll("path.link") + .data(links, d => { return d.target.id; }); + // Enter any new links at the parent's previous position. + link.enter().insert("path", "g") + .attr("class", "link") + .attr("d", d => { + let o = {x: source.x0, y: source.y0}; + return diagonal({source: o, target: o}); + }); + // Transition links to their new position. + link.transition() + .duration(duration) + .attr("d", diagonal); + // Transition exiting nodes to the parent's new position. + link.exit().transition() + .duration(duration) + .attr("d", d => { + let o = {x: source.x, y: source.y}; + return diagonal({source: o, target: o}); + }) + .remove(); + // Stash the old positions for transition. + nodes.forEach(d => { + d.x0 = d.x; + d.y0 = d.y; + }); + } + // Toggle children on click. + function click(d) { + if (d.children) { + d._children = d.children; + d.children = null; + } else { + d.children = d._children; + d._children = null; + } + update(d); + if (d.tool_id !== undefined && d.tool_id !== "undefined" && d.tool_id !== null && d.tool_id !== "") { + document.location.href = `${getAppRoot()}` + 'tool_runner?tool_id=' + d.tool_id; + } + } + function collapse(d) { + if (d.children) { + d._children = d.children; + d._children.forEach(collapse); + d.children = null; + } + } + d3.select(self.frameElement).style("height", "400px"); + root = predicted_data; + root.x0 = height / 2; + root.y0 = 0; + root.children.forEach(collapse); + update(root); + } +}); + +export default { + ToolRecommendationView: ToolRecommendationView +}; diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js index 89bdb4d3f28..93e7f2bc3f1 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form.js +++ b/client/galaxy/scripts/mvc/tool/tool-form.js @@ -13,6 +13,7 @@ import ToolFormBase from "mvc/tool/tool-form-base"; import Webhooks from "mvc/webhooks"; import Vue from "vue"; import ToolEntryPoints from "components/ToolEntryPoints/ToolEntryPoints"; +import ToolRecommendation from "mvc/tool-recommendation"; const View = Backbone.View.extend({ initialize: function(options) { @@ -267,6 +268,14 @@ const View = Backbone.View.extend({ } } this.$el.append(this._templateSuccess(response, job_def)); + let enable_tool_recommendations = window.Galaxy.config.enable_tool_recommendations; + if (enable_tool_recommendations === true || enable_tool_recommendations === 'true') { + // show tool recommendations + this.$el.append($("
", { id: "tool-recommendation-view" })); + var toolRecommendation = new ToolRecommendation.ToolRecommendationView({ + toolId: job_def.tool_id + }); + } this.$el.parent().scrollTop(0); // Show Webhook if job is running if (response.jobs && response.jobs.length > 0) { diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index ed488b4f892..c6b0e3f2e19 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1650,3 +1650,23 @@ body.reports { bottom: 0; background: url(../../images/largespinner.gif) no-repeat center center fixed; } + +/* For collapsible D3 tree */ + +.node { + cursor: pointer; +} +.node circle { + fill: #000; + stroke: #c2ebc2; + stroke-width: 0.3rem; +} +.node text { + font: 0.75rem sans-serif; +} + +.link { + fill: none; + stroke: #c2ebc2; + stroke-width: 0.3rem; +} diff --git a/config/tool_recommendations_overwrite.yml.sample b/config/tool_recommendations_overwrite.yml.sample new file mode 100644 index 00000000000..b1fc424a766 --- /dev/null +++ b/config/tool_recommendations_overwrite.yml.sample @@ -0,0 +1,48 @@ +# Provide a list of tools which are deprecated. These tools would be removed the list of +# recommended tools by the deep learning model. These tools will be removed from the recommendations and a warning message, set as the 'text_message', +# is shown instead of their recommendations and when they are executed. +# Format: +# <>: +# - is_deprecate: True +# - text_message: <> +# +# For the following example, the tool 'cufflinks' is deprecated. It will be removed from the recommendations and a warning message, set as the 'text_message', +# is shown instead of its recommendations and when it is executed: +# +# cufflinks: +# - is_deprecated: True +# text_message: It is deprecated +# +# +# +# Provide list of tools to be recommended. These tools will either be appended to the tools recommended by deep learning model or +# completely overwrite them with these tools. +# Format: +# <>: +# - tool_id: <> +# name: <> +# i_extensions: +# - <> +# - <> +# - tool_id: <> +# name: <> +# i_extensions: +# - <> +# - <> +# <>: +# - tool_id ... +# +# For the following example, the tools 'cat1' and 'sort1' are shown as the recommendations for tool 'Filter1': +# +# Filter1: +# - tool_id: 'cat1' +# name: 'Concatenate datasets tail-to-head' +# i_extensions: +# - tabular +# - txt +# - tool_id: 'sort1' +# name: 'Sort data in ascending or descending order' +# i_extensions: +# - tabular +# - txt + diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index b26ccd2a90d..fdc5c469349 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -4011,4 +4011,68 @@ :Type: int +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``enable_tool_recommendations`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Allow the display of tool recommendations in workflow editor and + after tool execution. If it is enabled and set to true, please + enable 'tool_recommendation_model_path' as well +:Default: ``false`` +:Type: bool + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``tool_recommendation_model_path`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Set remote path of the trained model (HDF5 file) for tool + recommendation. It is required to be set if + 'enable_tool_recommendations' property is enabled and set to true. +:Default: ``https://github.com/anuprulez/download_store/raw/tool_recommendation_model/tool_recommendation_model/tool_recommendation_model.hdf5`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~ +``topk_recommendations`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Set the number of predictions/recommendations to be made by the + model +:Default: ``20`` +:Type: int + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``admin_tool_recommendations_path`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Set path to the additional tool preferences from Galaxy admins. It + has two blocks. One for listing deprecated tools which will be + removed from the recommendations and another is for adding + additional tools to be recommended along side those from the deep + learning model. +:Default: ``None`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``overwrite_model_recommendations`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Overwrite or append to the tool recommendations by the deep + learning model. When set to true, all the recommendations by the + deep learning model are overwritten by the recommendations set by + an admin in a config file 'tool_recommendations_overwrite.yml'. + When set to false, the recommended tools by admins and predicted + by the deep learning model are shown. +:Default: ``false`` +:Type: bool + + diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index cc3d9ed3569..b57de1ff129 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1960,3 +1960,32 @@ galaxy: # as threshold (above threshold: regular select fields will be used) #select_type_workflow_threshold: -1 + # Allow the display of tool recommendations in workflow editor and + # after tool execution. If it is enabled and set to true, please + # enable 'tool_recommendation_model_path' as well + #enable_tool_recommendations: false + + # Set remote path of the trained model (HDF5 file) for tool + # recommendation. It is required to be set if + # 'enable_tool_recommendations' property is enabled and set to true. + #tool_recommendation_model_path: https://github.com/anuprulez/download_store/raw/tool_recommendation_model/tool_recommendation_model/tool_recommendation_model.hdf5 + + # Set the number of predictions/recommendations to be made by the + # model + #topk_recommendations: 20 + + # Set path to the additional tool preferences from Galaxy admins. It + # has two blocks. One for listing deprecated tools which will be + # removed from the recommendations and another is for adding + # additional tools to be recommended along side those from the deep + # learning model. + #admin_tool_recommendations_path: null + + # Overwrite or append to the tool recommendations by the deep learning + # model. When set to true, all the recommendations by the deep + # learning model are overwritten by the recommendations set by an + # admin in a config file 'tool_recommendations_overwrite.yml'. When + # set to false, the recommended tools by admins and predicted by the + # deep learning model are shown. + #overwrite_model_recommendations: false + diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 61ceec85d97..183dfb77fc8 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -78,6 +78,7 @@ jmespath==0.9.4 jsonpatch==1.25 jsonpointer==2.0 jsonschema==3.2.0 +keras==2.2.4 keystoneauth1==3.18.0 kombu==4.6.7 lockfile==0.12.2 @@ -180,6 +181,7 @@ subprocess32==3.5.4 ; python_version < '3.0' svgwrite==1.3.1 tempita==0.5.2 tenacity==4.12.0 +tensorflow==1.12.2 typing-extensions==3.7.4.1 typing==3.7.4.1 ; python_version < '3.5' tzlocal==2.0.0 diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 6df4ace6bbe..4812e33091f 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -62,6 +62,11 @@ class ConfigSerializer(base.ModelSerializer): 'communication_server_port' : _use_config, 'communication_server_host' : _use_config, 'persistent_communication_rooms' : _use_config, + 'enable_tool_recommendations' : _use_config, + 'tool_recommendation_model_path' : _use_config, + 'admin_tool_recommendations_path' : _use_config, + 'overwrite_model_recommendations' : _use_config, + 'topk_recommendations' : _use_config, 'allow_user_impersonation' : _use_config, 'allow_user_creation' : _defaults_to(False), # schema default is True 'use_remote_user' : _defaults_to(None), # schema default is False; or config.single_user diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 15ed31f2f86..5a5fbb594ff 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -14,6 +14,10 @@ from markupsafe import escape from sqlalchemy import desc, false, or_, true from sqlalchemy.orm import joinedload +import numpy as np +import h5py +from keras.models import model_from_json + from galaxy import ( exceptions, model, @@ -57,6 +61,10 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn self.history_manager = histories.HistoryManager(app) self.workflow_manager = workflows.WorkflowsManager(app) self.workflow_contents_manager = workflows.WorkflowContentsManager(app) + self.tool_recommendation_model_path = None + self.admin_tool_recommendations_path = None + self.deprecated_tools = dict() + self.admin_recommendations = dict() def __get_full_shed_url(self, url): for name, shed_url in self.app.tool_shed_registry.tool_sheds.items(): @@ -631,6 +639,222 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn 'post_job_actions' : module.get_post_job_actions(inputs) } + @expose_api + def get_tool_predictions(self, trans, payload, **kwd): + """ + POST /api/workflows/get_tool_predictions + Fetch predicted tools for a workflow + :type payload: dict + :param payload: a dictionary containing two parameters: + 'tool_sequence' - comma separated sequence of tool ids + 'remote_model_url' - (optional) path to the deep learning model + """ + remote_model_url = payload.get('remote_model_url', None) + if remote_model_url is None: + remote_model_url = trans.app.config.tool_recommendation_model_path + if 'tool_sequence' not in payload or remote_model_url is None: + return + tool_sequence = payload.get('tool_sequence', "") + # collect tool recommendation preferences if set by admin + self.__collect_admin_preferences(trans.app.config.admin_tool_recommendations_path) + # recreate the neural network model to be used for prediction + self.__set_model(trans, remote_model_url) + # get the recommended tools for a tool sequence + recommended_tools = self.__compute_tool_prediction(trans, tool_sequence) + return { + "current_tool": tool_sequence, + "predicted_data": recommended_tools + } + + # + # -- Helper methods -- + # + + def __set_model(self, trans, remote_model_url): + """ + Create model and associated dictionaries for recommendations + """ + if not self.tool_recommendation_model_path: + self.tool_recommendation_model_path = self.__download_model(remote_model_url) + self.all_tools = dict() + model_weights = list() + counter_layer_weights = 0 + # collect ids and names of all the installed tools + for tool_id, tool in trans.app.toolbox.tools(): + t_id_renamed = tool_id + if t_id_renamed.find("/") > -1: + t_id_renamed = t_id_renamed.split("/")[-2] + self.all_tools[t_id_renamed] = (tool_id, tool.name) + # read the hdf5 attributes + trained_model = h5py.File(self.tool_recommendation_model_path, 'r') + model_config = json.loads(trained_model.get('model_config').value) + self.loaded_model = model_from_json(model_config) + # set the dictionary of tools + self.model_data_dictionary = json.loads(trained_model.get('data_dictionary').value) + self.reverse_dictionary = dict((v, k) for k, v in self.model_data_dictionary.items()) + # set the list of compatible tools + self.compatible_tools = json.loads(trained_model.get('compatible_tools').value) + self.tool_weights = json.loads(trained_model.get('class_weights').value) + self.tool_weights_sorted = dict() + # sort the tools' usage dictionary + tool_pos_sorted = [int(key) for key in self.tool_weights.keys()] + for k in tool_pos_sorted: + self.tool_weights_sorted[k] = self.tool_weights[str(k)] + # iterate through all the attributes of the model to find weights of neural network layers + for item in trained_model.keys(): + if "weight_" in item: + d_key = "weight_" + str(counter_layer_weights) + weights = trained_model.get(d_key).value + model_weights.append(weights) + counter_layer_weights += 1 + # set the model weights + self.loaded_model.set_weights(model_weights) + + def __collect_admin_preferences(self, admin_path): + """ + Collect preferences for recommendations of tools + set by admins as dictionaries of deprecated tools and + additional recommendations + """ + if not self.admin_tool_recommendations_path and admin_path is not None: + self.admin_tool_recommendations_path = os.path.join(os.getcwd(), admin_path) + with open(self.admin_tool_recommendations_path) as admin_recommendations: + admin_recommendation_preferences = yaml.safe_load(admin_recommendations) + if admin_recommendation_preferences: + for tool_id in admin_recommendation_preferences: + tool_info = admin_recommendation_preferences[tool_id] + if 'is_deprecated' in tool_info[0]: + self.deprecated_tools[tool_id] = tool_info[0]["text_message"] + else: + if tool_id not in self.admin_recommendations: + self.admin_recommendations[tool_id] = tool_info + + def __download_model(self, model_url, download_local='database/'): + """ + Download the model from remote server + """ + local_dir = os.path.join(os.getcwd(), download_local, 'tool_recommendation_model.hdf5') + # read model from remote + model_binary = requests.get(model_url) + # save model to a local directory + with open(local_dir, 'wb') as model_file: + model_file.write(model_binary.content) + return local_dir + + def __get_tool_extensions(self, trans, tool_id): + """ + Get the input and output extensions of a tool + """ + payload = {'type': 'tool', 'tool_id': tool_id, '_': 'true'} + inputs = payload.get('inputs', {}) + trans.workflow_building_mode = workflow_building_modes.ENABLED + module = module_factory.from_dict(trans, payload) + if 'tool_state' not in payload: + module_state = {} + populate_state(trans, module.get_inputs(), inputs, module_state, check=False) + module.recover_state(module_state) + inputs = module.get_all_inputs(connectable_only=True) + outputs = module.get_all_outputs() + input_extensions = list() + output_extensions = list() + for i_ext in inputs: + input_extensions.extend(i_ext['extensions']) + for o_ext in outputs: + output_extensions.extend(o_ext['extensions']) + return input_extensions, output_extensions + + def __filter_tool_predictions(self, trans, prediction_data, tool_ids, tool_scores, last_tool_name): + """ + Filter tool predictions based on datatype compatibility and tool connections. + Add admin preferences to recommendations. + """ + last_compatible_tools = list() + if last_tool_name in self.compatible_tools: + last_compatible_tools = self.compatible_tools[last_tool_name].split(",") + prediction_data["is_deprecated"] = False + t_ids_scores = zip(tool_ids, tool_scores) + # form the payload of the predicted tools to be shown + for child, score in t_ids_scores: + c_dict = dict() + for t_id in self.all_tools: + # select the name and tool id if it is installed in Galaxy + if t_id == child and score > 0.0 and child in last_compatible_tools and child not in self.deprecated_tools: + full_tool_id = self.all_tools[t_id][0] + pred_input_extensions, _ = self.__get_tool_extensions(trans, full_tool_id) + c_dict["name"] = self.all_tools[t_id][1] + " (" + str(score) + "%)" + c_dict["tool_id"] = full_tool_id + c_dict["i_extensions"] = list(set(pred_input_extensions)) + prediction_data["children"].append(c_dict) + break + # incorporate preferences set by admins + if self.admin_tool_recommendations_path: + # filter out deprecated tools + t_ids_scores = [(tid, score) for tid, score in zip(tool_ids, tool_scores) if tid not in self.deprecated_tools] + # set the property if the last tool of the sequence is deprecated + if last_tool_name in self.deprecated_tools: + prediction_data["is_deprecated"] = True + prediction_data["message"] = self.deprecated_tools[last_tool_name] + # add the recommendations given by admins + for tool_id in self.admin_recommendations: + if last_tool_name == tool_id: + admin_recommendations = self.admin_recommendations[tool_id] + if trans.app.config.overwrite_model_recommendations is True: + prediction_data["children"] = admin_recommendations + else: + prediction_data["children"].extend(admin_recommendations) + break + # get the root name for displaying after tool run + for t_id in self.all_tools: + if t_id == last_tool_name: + prediction_data["name"] = self.all_tools[t_id][1] + break + return prediction_data + + def __compute_tool_prediction(self, trans, tool_sequence): + """ + Compute the predicted tools for a tool sequences + Return a payload with the tool sequences and recommended tools + Return an empty payload with just the tool sequence if anything goes wrong within the try block + """ + max_seq_len = 25 + topk = trans.app.config.topk_recommendations + prediction_data = dict() + tool_sequence = tool_sequence.split(",")[::-1] + prediction_data["name"] = ",".join(tool_sequence) + prediction_data["children"] = list() + last_tool_name = tool_sequence[-1] + # do prediction only if the last is present in the collections of tools + if last_tool_name in self.model_data_dictionary: + sample = np.zeros(max_seq_len) + # get the list of datatype extensions of the last tool of the tool sequence + _, last_output_extensions = self.__get_tool_extensions(trans, self.all_tools[last_tool_name][0]) + prediction_data["o_extensions"] = list(set(last_output_extensions)) + # get tool names without slashes and create a sequence vector + for idx, tool_name in enumerate(tool_sequence): + if tool_name.find("/") > -1: + tool_name = tool_name.split("/")[-2] + sample[idx] = int(self.model_data_dictionary[tool_name]) + sample = np.reshape(sample, (1, max_seq_len)) + # predict next tools for a test path + prediction = self.loaded_model.predict(sample, verbose=0) + prediction = np.reshape(prediction, (prediction.shape[1],)) + # boost the predicted scores using tools' usage + weight_values = list(self.tool_weights_sorted.values()) + prediction = prediction * weight_values + # normalize the predicted scores with max and sort the predictions + max_prediction = float(np.max(prediction)) + if max_prediction == 0.0: + max_prediction = 1.0 + prediction = prediction / max_prediction + prediction_pos = np.argsort(prediction, axis=-1) + # get topk prediction + topk_prediction_pos = prediction_pos[-topk:] + # read tool names using reverse dictionary + pred_tool_ids = [self.reverse_dictionary[int(tool_pos)] for tool_pos in topk_prediction_pos] + predicted_scores = [int(prediction[pos] * 100) for pos in topk_prediction_pos] + prediction_data = self.__filter_tool_predictions(trans, prediction_data, pred_tool_ids[::-1], predicted_scores[::-1], last_tool_name) + return prediction_data + # # -- Helper methods -- # diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 7281d8240af..90485e2b23e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -402,6 +402,7 @@ def populate_api_routes(webapp, app): webapp.mapper.connect('/api/container_resolvers/{index}/toolbox', action="resolve_toolbox", controller="container_resolution", conditions=dict(method=["GET"])) webapp.mapper.connect('/api/container_resolvers/{index}/resolve/install', action="resolve_with_install", controller="container_resolution", conditions=dict(method=["POST"])) webapp.mapper.connect('/api/container_resolvers/{index}/toolbox/install', action="resolve_toolbox_with_install", controller="container_resolution", conditions=dict(method=["POST"])) + webapp.mapper.connect('/api/workflows/get_tool_predictions', action='get_tool_predictions', controller="workflows", conditions=dict(method=["POST"])) webapp.mapper.resource_with_deleted('user', 'users', path_prefix='/api') webapp.mapper.resource('genome', 'genomes', path_prefix='/api') diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index e21eb5f8ea7..bc646a13b63 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2959,3 +2959,43 @@ mapping: use 0 in order to always use select2 fields, use -1 (default) in order to always use the regular select fields, use any other positive number as threshold (above threshold: regular select fields will be used) + + enable_tool_recommendations: + type: bool + default: false + required: false + desc: | + Allow the display of tool recommendations in workflow editor and after tool execution. + If it is enabled and set to true, please enable 'tool_recommendation_model_path' as well + + tool_recommendation_model_path: + type: str + default: 'https://github.com/anuprulez/download_store/raw/tool_recommendation_model/tool_recommendation_model/tool_recommendation_model.hdf5' + required: false + desc: | + Set remote path of the trained model (HDF5 file) for tool recommendation. + It is required to be set if 'enable_tool_recommendations' property is enabled and set to true. + + topk_recommendations: + type: int + default: 20 + required: false + desc: | + Set the number of predictions/recommendations to be made by the model + + admin_tool_recommendations_path: + type: str + required: false + desc: | + Set path to the additional tool preferences from Galaxy admins. + It has two blocks. One for listing deprecated tools which will be removed from the recommendations and + another is for adding additional tools to be recommended along side those from the deep learning model. + + overwrite_model_recommendations: + type: bool + default: false + required: false + desc: | + Overwrite or append to the tool recommendations by the deep learning model. When set to true, all the recommendations by the deep learning model + are overwritten by the recommendations set by an admin in a config file 'tool_recommendations_overwrite.yml'. When set to false, the recommended tools + by admins and predicted by the deep learning model are shown. diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index a40ab737f8a..5e957cec22c 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -331,6 +331,24 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): self._assert_user_has_workflow_with_name(name) return upload_response + def test_get_tool_predictions(self): + request = {"tool_sequence": "Cut1", "remote_model_url": "https://github.com/anuprulez/download_store/raw/tool_recommendation_model/tool_recommendation_model/tool_recommendation_model.hdf5"} + actual_recommendations = ['Filter1', 'cat1', 'addValue', 'comp1', 'Grep1'] + route = "workflows/get_tool_predictions" + response = self._post(route, data=request) + recommendation_response = response.json() + # check Ok response from the API + self._assert_status_code_is(response, 200) + # check the input tool sequence + assert recommendation_response["current_tool"] == request["tool_sequence"] + # check non-empty predictions list + predicted_tools = recommendation_response["predicted_data"]["children"] + assert len(predicted_tools) > 0 + # check for the correct predictions + for tool in predicted_tools: + assert tool["tool_id"] in actual_recommendations + break + def test_update(self): original_workflow = self.workflow_populator.load_workflow(name="test_import") uuids = {} From dd09df0ab9b500d84c1cabec8eeb478dce8c147c Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 25 Feb 2020 18:22:08 +0100 Subject: [PATCH 02/38] Update tensorflow requirements --- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 183dfb77fc8..dc64eec6383 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -181,7 +181,7 @@ subprocess32==3.5.4 ; python_version < '3.0' svgwrite==1.3.1 tempita==0.5.2 tenacity==4.12.0 -tensorflow==1.12.2 +tensorflow==1.15.0 typing-extensions==3.7.4.1 typing==3.7.4.1 ; python_version < '3.5' tzlocal==2.0.0 From 915b3afa61285235a8df416f97490ad2bf67ffde Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 25 Feb 2020 18:32:39 +0100 Subject: [PATCH 03/38] Fix deprecation warning for H5py --- lib/galaxy/webapps/galaxy/api/workflows.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 5a5fbb594ff..62a67c17b3e 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -687,14 +687,14 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn self.all_tools[t_id_renamed] = (tool_id, tool.name) # read the hdf5 attributes trained_model = h5py.File(self.tool_recommendation_model_path, 'r') - model_config = json.loads(trained_model.get('model_config').value) + model_config = json.loads(trained_model['model_config'].value) self.loaded_model = model_from_json(model_config) # set the dictionary of tools - self.model_data_dictionary = json.loads(trained_model.get('data_dictionary').value) + self.model_data_dictionary = json.loads(trained_model['data_dictionary'].value) self.reverse_dictionary = dict((v, k) for k, v in self.model_data_dictionary.items()) # set the list of compatible tools - self.compatible_tools = json.loads(trained_model.get('compatible_tools').value) - self.tool_weights = json.loads(trained_model.get('class_weights').value) + self.compatible_tools = json.loads(trained_model['compatible_tools'].value) + self.tool_weights = json.loads(trained_model['class_weights'].value) self.tool_weights_sorted = dict() # sort the tools' usage dictionary tool_pos_sorted = [int(key) for key in self.tool_weights.keys()] @@ -704,7 +704,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn for item in trained_model.keys(): if "weight_" in item: d_key = "weight_" + str(counter_layer_weights) - weights = trained_model.get(d_key).value + weights = trained_model[d_key].value model_weights.append(weights) counter_layer_weights += 1 # set the model weights From 35ae3f375beae5e925be10eb774899ef14b8e6f2 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 26 Feb 2020 10:56:28 +0100 Subject: [PATCH 04/38] Fix linting issues --- .../galaxy/scripts/mvc/tool-recommendation.js | 67 +++++++++---------- client/galaxy/scripts/mvc/tool/tool-form.js | 4 +- lib/galaxy/webapps/galaxy/api/workflows.py | 5 +- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/client/galaxy/scripts/mvc/tool-recommendation.js b/client/galaxy/scripts/mvc/tool-recommendation.js index 9751d21db29..2e06e89b724 100644 --- a/client/galaxy/scripts/mvc/tool-recommendation.js +++ b/client/galaxy/scripts/mvc/tool-recommendation.js @@ -10,9 +10,9 @@ var ToolRecommendationView = Backbone.View.extend({ initialize: function(options) { let toolId = options.toolId || ""; - let self = this; + const self = this; if (toolId.indexOf("/") > 0) { - let toolIdSlash = toolId.split("/"); + const toolIdSlash = toolId.split("/"); toolId = toolIdSlash[toolIdSlash.length - 2]; } Utils.request({ @@ -21,26 +21,26 @@ var ToolRecommendationView = Backbone.View.extend({ data: {"tool_sequence": toolId}, success: data => { // get datatypes mapping - let datatypes_mapping = JSON.parse( + const datatypes_mapping = JSON.parse( $.ajax({ url: `${getAppRoot()}api/datatypes/mapping`, async: false }).responseText ); - let extToType = datatypes_mapping.ext_to_class_name; - let typeToType = datatypes_mapping.class_to_classes; - let predData = data.predicted_data; + const extToType = datatypes_mapping.ext_to_class_name; + const typeToType = datatypes_mapping.class_to_classes; + const predData = data.predicted_data; if (data !== null && predData.children.length > 0) { - let filteredData = {}; - let compatibleTools = {}; - let filteredChildren = []; - let outputDatatypes = predData["o_extensions"]; - for (const [index, nameObj] of predData.children.entries()) { - let inputDatatypes = nameObj["i_extensions"]; + const filteredData = {}; + const compatibleTools = {}; + const filteredChildren = []; + const outputDatatypes = predData["o_extensions"]; + for (const [_, nameObj] of predData.children.entries()) { + const inputDatatypes = nameObj["i_extensions"]; for (const out_t of outputDatatypes.entries()) { for(const in_t of inputDatatypes.entries()) { - let child = extToType[out_t[1]]; - let parent = extToType[in_t[1]]; + const child = extToType[out_t[1]]; + const parent = extToType[in_t[1]]; if (((typeToType[child] && parent in typeToType[child]) === true) || out_t[1] === "input" || out_t[1] === "_sniff_" || @@ -51,8 +51,8 @@ var ToolRecommendationView = Backbone.View.extend({ } } } - for (let id in compatibleTools) { - for (const [index, nameObj] of predData.children.entries()) { + for (const id in compatibleTools) { + for (const [_, nameObj] of predData.children.entries()) { if (nameObj["tool_id"] === id) { filteredChildren.push(nameObj); break @@ -75,33 +75,32 @@ var ToolRecommendationView = Backbone.View.extend({ }, render_tree: function(predicted_data) { - let margin = {top: 20, right: 30, bottom: 20, left: 250}, + const margin = {top: 20, right: 30, bottom: 20, left: 250}, width = 900 - margin.right - margin.left, - height = 300 - margin.top - margin.bottom; - let i = 0, - duration = 750, - root; - let tree = d3.layout.tree() + height = 300 - margin.top - margin.bottom, + duration = 750; + const tree = d3.layout.tree() .size([height, width]); - let diagonal = d3.svg.diagonal() - .projection(d => { return [d.y, d.x]; }); - let svg = d3.select("#tool-recommendation-view").append("svg") + const diagonal = d3.svg.diagonal() + .projection(d => { return [d.y, d.x]; }) + const svg = d3.select("#tool-recommendation-view").append("svg") .attr("width", width + margin.right + margin.left) .attr("height", height + margin.top + margin.bottom) .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - + let i = 0, + root = null; function update(source) { // Compute the new tree layout. - let nodes = tree.nodes(root).reverse(), - links = tree.links(nodes); + const nodes = tree.nodes(root).reverse(); + const links = tree.links(nodes); // Normalize for fixed-depth. nodes.forEach(d => { d.y = d.depth * 180; }); // Update the nodes… - let node = svg.selectAll("g.node") + const node = svg.selectAll("g.node") .data(nodes, d => { return d.id || (d.id = ++i); }); // Enter any new nodes at the parent's previous position. - let nodeEnter = node.enter().append("g") + const nodeEnter = node.enter().append("g") .attr("class", "node") .attr("transform", d => { return "translate(" + source.y0 + "," + source.x0 + ")"; }) .on("click", click); @@ -117,7 +116,7 @@ var ToolRecommendationView = Backbone.View.extend({ nodeEnter.append("title") .text(d => { return d.children || d._children ? "Click to collapse" : "Click to open tool definition"; }) // Transition nodes to their new position. - let nodeUpdate = node.transition() + const nodeUpdate = node.transition() .duration(duration) .attr("transform", d => { return "translate(" + d.y + "," + d.x + ")"; }); nodeUpdate.select("circle") @@ -126,7 +125,7 @@ var ToolRecommendationView = Backbone.View.extend({ nodeUpdate.select("text") .style("fill-opacity", 1); // Transition exiting nodes to the parent's new position. - let nodeExit = node.exit().transition() + const nodeExit = node.exit().transition() .duration(duration) .attr("transform", d => { return "translate(" + source.y + "," + source.x + ")"; }) .remove(); @@ -135,7 +134,7 @@ var ToolRecommendationView = Backbone.View.extend({ nodeExit.select("text") .style("fill-opacity", 1e-6); // Update the links… - let link = svg.selectAll("path.link") + const link = svg.selectAll("path.link") .data(links, d => { return d.target.id; }); // Enter any new links at the parent's previous position. link.enter().insert("path", "g") @@ -152,7 +151,7 @@ var ToolRecommendationView = Backbone.View.extend({ link.exit().transition() .duration(duration) .attr("d", d => { - let o = {x: source.x, y: source.y}; + const o = {x: source.x, y: source.y}; return diagonal({source: o, target: o}); }) .remove(); diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js index 93e7f2bc3f1..3cd3e4ddc70 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form.js +++ b/client/galaxy/scripts/mvc/tool/tool-form.js @@ -268,11 +268,11 @@ const View = Backbone.View.extend({ } } this.$el.append(this._templateSuccess(response, job_def)); - let enable_tool_recommendations = window.Galaxy.config.enable_tool_recommendations; + const enable_tool_recommendations = window.Galaxy.config.enable_tool_recommendations; if (enable_tool_recommendations === true || enable_tool_recommendations === 'true') { // show tool recommendations this.$el.append($("
", { id: "tool-recommendation-view" })); - var toolRecommendation = new ToolRecommendation.ToolRecommendationView({ + const toolRecommendation = new ToolRecommendation.ToolRecommendationView({ toolId: job_def.tool_id }); } diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 62a67c17b3e..c8b8e04d808 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import io import json +import yaml import logging import os @@ -12,10 +13,10 @@ import requests from gxformat2._yaml import ordered_dump from markupsafe import escape from sqlalchemy import desc, false, or_, true +import h5py +import numpy as np from sqlalchemy.orm import joinedload -import numpy as np -import h5py from keras.models import model_from_json from galaxy import ( From cf54ae1bebcf9c8aaea67ce634e887794777752d Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 26 Feb 2020 12:29:46 +0100 Subject: [PATCH 05/38] Fix linting issues --- .../galaxy/scripts/mvc/tool-recommendation.js | 39 +++++++------- client/galaxy/scripts/mvc/tool/tool-form.js | 2 +- doc/source/admin/galaxy_options.rst | 2 +- lib/galaxy/config/sample/galaxy.yml.sample | 2 +- lib/galaxy/webapps/galaxy/api/workflows.py | 51 ++++++++++--------- lib/galaxy/webapps/galaxy/config_schema.yml | 2 + 6 files changed, 51 insertions(+), 47 deletions(-) diff --git a/client/galaxy/scripts/mvc/tool-recommendation.js b/client/galaxy/scripts/mvc/tool-recommendation.js index 2e06e89b724..b2254a573b6 100644 --- a/client/galaxy/scripts/mvc/tool-recommendation.js +++ b/client/galaxy/scripts/mvc/tool-recommendation.js @@ -26,35 +26,36 @@ var ToolRecommendationView = Backbone.View.extend({ url: `${getAppRoot()}api/datatypes/mapping`, async: false }).responseText - ); - const extToType = datatypes_mapping.ext_to_class_name; - const typeToType = datatypes_mapping.class_to_classes; - const predData = data.predicted_data; + ), + extToType = datatypes_mapping.ext_to_class_name, + typeToType = datatypes_mapping.class_to_classes, + predData = data.predicted_data; if (data !== null && predData.children.length > 0) { - const filteredData = {}; - const compatibleTools = {}; - const filteredChildren = []; - const outputDatatypes = predData["o_extensions"]; - for (const [_, nameObj] of predData.children.entries()) { - const inputDatatypes = nameObj["i_extensions"]; + const filteredData = {}, + compatibleTools = {}, + filteredChildren = [], + outputDatatypes = predData["o_extensions"], + children = predData.children; + for (const nameObj of children.entries()) { + const inputDatatypes = nameObj[1]["i_extensions"]; for (const out_t of outputDatatypes.entries()) { for(const in_t of inputDatatypes.entries()) { - const child = extToType[out_t[1]]; - const parent = extToType[in_t[1]]; + const child = extToType[out_t[1]], + parent = extToType[in_t[1]]; if (((typeToType[child] && parent in typeToType[child]) === true) || out_t[1] === "input" || out_t[1] === "_sniff_" || out_t[1] === "input_collection") { - compatibleTools[nameObj["tool_id"]] = nameObj["name"]; + compatibleTools[nameObj[1]["tool_id"]] = nameObj[1]["name"]; break } } } } for (const id in compatibleTools) { - for (const [_, nameObj] of predData.children.entries()) { - if (nameObj["tool_id"] === id) { - filteredChildren.push(nameObj); + for (const nameObj of children.entries()) { + if (nameObj[1]["tool_id"] === id) { + filteredChildren.push(nameObj[1]); break } } @@ -92,8 +93,8 @@ var ToolRecommendationView = Backbone.View.extend({ root = null; function update(source) { // Compute the new tree layout. - const nodes = tree.nodes(root).reverse(); - const links = tree.links(nodes); + const nodes = tree.nodes(root).reverse(), + links = tree.links(nodes); // Normalize for fixed-depth. nodes.forEach(d => { d.y = d.depth * 180; }); // Update the nodes… @@ -140,7 +141,7 @@ var ToolRecommendationView = Backbone.View.extend({ link.enter().insert("path", "g") .attr("class", "link") .attr("d", d => { - let o = {x: source.x0, y: source.y0}; + const o = {x: source.x0, y: source.y0}; return diagonal({source: o, target: o}); }); // Transition links to their new position. diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js index 3cd3e4ddc70..82c7dc82034 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form.js +++ b/client/galaxy/scripts/mvc/tool/tool-form.js @@ -272,7 +272,7 @@ const View = Backbone.View.extend({ if (enable_tool_recommendations === true || enable_tool_recommendations === 'true') { // show tool recommendations this.$el.append($("
", { id: "tool-recommendation-view" })); - const toolRecommendation = new ToolRecommendation.ToolRecommendationView({ + new ToolRecommendation.ToolRecommendationView({ toolId: job_def.tool_id }); } diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index fdc5c469349..4920d8ae8f9 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -4056,7 +4056,7 @@ removed from the recommendations and another is for adding additional tools to be recommended along side those from the deep learning model. -:Default: ``None`` +:Default: ``tool_recommendations_overwrite.yml`` :Type: str diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index b57de1ff129..41310e3d20c 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1979,7 +1979,7 @@ galaxy: # removed from the recommendations and another is for adding # additional tools to be recommended along side those from the deep # learning model. - #admin_tool_recommendations_path: null + #admin_tool_recommendations_path: tool_recommendations_overwrite.yml # Overwrite or append to the tool recommendations by the deep learning # model. When set to true, all the recommendations by the deep diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index c8b8e04d808..32186d785f6 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -5,20 +5,11 @@ from __future__ import absolute_import import io import json -import yaml import logging import os import requests -from gxformat2._yaml import ordered_dump -from markupsafe import escape -from sqlalchemy import desc, false, or_, true -import h5py -import numpy as np -from sqlalchemy.orm import joinedload - -from keras.models import model_from_json - +import yaml from galaxy import ( exceptions, model, @@ -46,11 +37,20 @@ from galaxy.webapps.base.controller import ( url_for, UsesStoredWorkflowMixin ) + from galaxy.workflow.extract import extract_workflow from galaxy.workflow.modules import module_factory from galaxy.workflow.reports import generate_report from galaxy.workflow.run import invoke, queue_invoke from galaxy.workflow.run_request import build_workflow_run_configs +from gxformat2._yaml import ordered_dump +from markupsafe import escape + +import h5py +import numpy as np +from keras.models import model_from_json +from sqlalchemy import desc, false, or_, true +from sqlalchemy.orm import joinedload log = logging.getLogger(__name__) @@ -688,14 +688,14 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn self.all_tools[t_id_renamed] = (tool_id, tool.name) # read the hdf5 attributes trained_model = h5py.File(self.tool_recommendation_model_path, 'r') - model_config = json.loads(trained_model['model_config'].value) + model_config = json.loads(trained_model['model_config'][()]) self.loaded_model = model_from_json(model_config) # set the dictionary of tools - self.model_data_dictionary = json.loads(trained_model['data_dictionary'].value) + self.model_data_dictionary = json.loads(trained_model['data_dictionary'][()]) self.reverse_dictionary = dict((v, k) for k, v in self.model_data_dictionary.items()) # set the list of compatible tools - self.compatible_tools = json.loads(trained_model['compatible_tools'].value) - self.tool_weights = json.loads(trained_model['class_weights'].value) + self.compatible_tools = json.loads(trained_model['compatible_tools'][()]) + self.tool_weights = json.loads(trained_model['class_weights'][()]) self.tool_weights_sorted = dict() # sort the tools' usage dictionary tool_pos_sorted = [int(key) for key in self.tool_weights.keys()] @@ -705,7 +705,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn for item in trained_model.keys(): if "weight_" in item: d_key = "weight_" + str(counter_layer_weights) - weights = trained_model[d_key].value + weights = trained_model[d_key][()] model_weights.append(weights) counter_layer_weights += 1 # set the model weights @@ -719,16 +719,17 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn """ if not self.admin_tool_recommendations_path and admin_path is not None: self.admin_tool_recommendations_path = os.path.join(os.getcwd(), admin_path) - with open(self.admin_tool_recommendations_path) as admin_recommendations: - admin_recommendation_preferences = yaml.safe_load(admin_recommendations) - if admin_recommendation_preferences: - for tool_id in admin_recommendation_preferences: - tool_info = admin_recommendation_preferences[tool_id] - if 'is_deprecated' in tool_info[0]: - self.deprecated_tools[tool_id] = tool_info[0]["text_message"] - else: - if tool_id not in self.admin_recommendations: - self.admin_recommendations[tool_id] = tool_info + if os.path.exists(self.admin_tool_recommendations_path): + with open(self.admin_tool_recommendations_path) as admin_recommendations: + admin_recommendation_preferences = yaml.safe_load(admin_recommendations) + if admin_recommendation_preferences: + for tool_id in admin_recommendation_preferences: + tool_info = admin_recommendation_preferences[tool_id] + if 'is_deprecated' in tool_info[0]: + self.deprecated_tools[tool_id] = tool_info[0]["text_message"] + else: + if tool_id not in self.admin_recommendations: + self.admin_recommendations[tool_id] = tool_info def __download_model(self, model_url, download_local='database/'): """ diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index bc646a13b63..3fdd1ba0fee 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2986,6 +2986,8 @@ mapping: admin_tool_recommendations_path: type: str required: false + default: 'tool_recommendations_overwrite.yml' + path_resolves_to: config_dir desc: | Set path to the additional tool preferences from Galaxy admins. It has two blocks. One for listing deprecated tools which will be removed from the recommendations and From e409cdbc9de76c22bbb9adce752ca349f18def0a Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 26 Feb 2020 14:17:44 +0100 Subject: [PATCH 06/38] Run prettier --- .../galaxy/scripts/mvc/tool-recommendation.js | 162 +++++++++++------- client/galaxy/scripts/mvc/tool/tool-form.js | 2 +- 2 files changed, 102 insertions(+), 62 deletions(-) diff --git a/client/galaxy/scripts/mvc/tool-recommendation.js b/client/galaxy/scripts/mvc/tool-recommendation.js index b2254a573b6..09e9a29812f 100644 --- a/client/galaxy/scripts/mvc/tool-recommendation.js +++ b/client/galaxy/scripts/mvc/tool-recommendation.js @@ -4,7 +4,6 @@ import * as d3 from "d3"; import { getAppRoot } from "onload/loadConfig"; import $ from "jquery"; - var ToolRecommendationView = Backbone.View.extend({ el: "#tool-recommendation-view", @@ -18,18 +17,18 @@ var ToolRecommendationView = Backbone.View.extend({ Utils.request({ type: "POST", url: `${getAppRoot()}api/workflows/get_tool_predictions`, - data: {"tool_sequence": toolId}, + data: { tool_sequence: toolId }, success: data => { // get datatypes mapping const datatypes_mapping = JSON.parse( - $.ajax({ - url: `${getAppRoot()}api/datatypes/mapping`, - async: false - }).responseText - ), - extToType = datatypes_mapping.ext_to_class_name, - typeToType = datatypes_mapping.class_to_classes, - predData = data.predicted_data; + $.ajax({ + url: `${getAppRoot()}api/datatypes/mapping`, + async: false + }).responseText + ), + extToType = datatypes_mapping.ext_to_class_name, + typeToType = datatypes_mapping.class_to_classes, + predData = data.predicted_data; if (data !== null && predData.children.length > 0) { const filteredData = {}, compatibleTools = {}, @@ -39,15 +38,17 @@ var ToolRecommendationView = Backbone.View.extend({ for (const nameObj of children.entries()) { const inputDatatypes = nameObj[1]["i_extensions"]; for (const out_t of outputDatatypes.entries()) { - for(const in_t of inputDatatypes.entries()) { + for (const in_t of inputDatatypes.entries()) { const child = extToType[out_t[1]], parent = extToType[in_t[1]]; - if (((typeToType[child] && parent in typeToType[child]) === true) || - out_t[1] === "input" || - out_t[1] === "_sniff_" || - out_t[1] === "input_collection") { + if ( + (typeToType[child] && parent in typeToType[child]) === true || + out_t[1] === "input" || + out_t[1] === "_sniff_" || + out_t[1] === "input_collection" + ) { compatibleTools[nameObj[1]["tool_id"]] = nameObj[1]["name"]; - break + break; } } } @@ -56,7 +57,7 @@ var ToolRecommendationView = Backbone.View.extend({ for (const nameObj of children.entries()) { if (nameObj[1]["tool_id"] === id) { filteredChildren.push(nameObj[1]); - break + break; } } } @@ -64,11 +65,20 @@ var ToolRecommendationView = Backbone.View.extend({ filteredData["name"] = predData["name"]; filteredData["children"] = filteredChildren; if (filteredChildren.length > 0 && predData["is_deprecated"] === false) { - self.$el.append("
You have used " + filteredData.name + " tool. For further analysis, you could try using the following/recommended tools. The recommended tools are shown in the decreasing order of their scores predicted using machine learning analysis on workflows. A tool with a higher score (closer to 100%) may fit better as the following tool than a tool with a lower score. Please click on one of the following/recommended tools to open its definition.
"); + self.$el.append( + "
You have used " + + filteredData.name + + " tool. For further analysis, you could try using the following/recommended tools. The recommended tools are shown in the decreasing order of their scores predicted using machine learning analysis on workflows. A tool with a higher score (closer to 100%) may fit better as the following tool than a tool with a lower score. Please click on one of the following/recommended tools to open its definition.
" + ); self.render_tree(filteredData); - } - else if(predData["is_deprecated"] === true) { - self.$el.append("
You have used " + predData.name + " tool. " + predData["message"] + ".
"); + } else if (predData["is_deprecated"] === true) { + self.$el.append( + "
You have used " + + predData.name + + " tool. " + + predData["message"] + + ".
" + ); } } } @@ -76,15 +86,17 @@ var ToolRecommendationView = Backbone.View.extend({ }, render_tree: function(predicted_data) { - const margin = {top: 20, right: 30, bottom: 20, left: 250}, + const margin = { top: 20, right: 30, bottom: 20, left: 250 }, width = 900 - margin.right - margin.left, height = 300 - margin.top - margin.bottom, duration = 750; - const tree = d3.layout.tree() - .size([height, width]); - const diagonal = d3.svg.diagonal() - .projection(d => { return [d.y, d.x]; }) - const svg = d3.select("#tool-recommendation-view").append("svg") + const tree = d3.layout.tree().size([height, width]); + const diagonal = d3.svg.diagonal().projection(d => { + return [d.y, d.x]; + }); + const svg = d3 + .select("#tool-recommendation-view") + .append("svg") .attr("width", width + margin.right + margin.left) .attr("height", height + margin.top + margin.bottom) .append("g") @@ -96,64 +108,92 @@ var ToolRecommendationView = Backbone.View.extend({ const nodes = tree.nodes(root).reverse(), links = tree.links(nodes); // Normalize for fixed-depth. - nodes.forEach(d => { d.y = d.depth * 180; }); + nodes.forEach(d => { + d.y = d.depth * 180; + }); // Update the nodes… - const node = svg.selectAll("g.node") - .data(nodes, d => { return d.id || (d.id = ++i); }); + const node = svg.selectAll("g.node").data(nodes, d => { + return d.id || (d.id = ++i); + }); // Enter any new nodes at the parent's previous position. - const nodeEnter = node.enter().append("g") + const nodeEnter = node + .enter() + .append("g") .attr("class", "node") - .attr("transform", d => { return "translate(" + source.y0 + "," + source.x0 + ")"; }) + .attr("transform", d => { + return "translate(" + source.y0 + "," + source.x0 + ")"; + }) .on("click", click); - nodeEnter.append("circle") + nodeEnter + .append("circle") .attr("r", 1e-6) - .style("fill", d => { return d._children ? "lightsteelblue" : "#fff"; }); - nodeEnter.append("text") - .attr("x", d => { return d.children || d._children ? -10 : 10; }) + .style("fill", d => { + return d._children ? "lightsteelblue" : "#fff"; + }); + nodeEnter + .append("text") + .attr("x", d => { + return d.children || d._children ? -10 : 10; + }) .attr("dy", ".35em") - .attr("text-anchor", d => { return d.children || d._children ? "end" : "start"; }) - .text(d => { return d.name; }) + .attr("text-anchor", d => { + return d.children || d._children ? "end" : "start"; + }) + .text(d => { + return d.name; + }) .style("fill-opacity", 1e-6); - nodeEnter.append("title") - .text(d => { return d.children || d._children ? "Click to collapse" : "Click to open tool definition"; }) + nodeEnter.append("title").text(d => { + return d.children || d._children ? "Click to collapse" : "Click to open tool definition"; + }); // Transition nodes to their new position. - const nodeUpdate = node.transition() + const nodeUpdate = node + .transition() .duration(duration) - .attr("transform", d => { return "translate(" + d.y + "," + d.x + ")"; }); - nodeUpdate.select("circle") + .attr("transform", d => { + return "translate(" + d.y + "," + d.x + ")"; + }); + nodeUpdate + .select("circle") .attr("r", 4.5) - .style("fill", d => { return d._children ? "lightsteelblue" : "#fff"; }); - nodeUpdate.select("text") - .style("fill-opacity", 1); + .style("fill", d => { + return d._children ? "lightsteelblue" : "#fff"; + }); + nodeUpdate.select("text").style("fill-opacity", 1); // Transition exiting nodes to the parent's new position. - const nodeExit = node.exit().transition() + const nodeExit = node + .exit() + .transition() .duration(duration) - .attr("transform", d => { return "translate(" + source.y + "," + source.x + ")"; }) + .attr("transform", d => { + return "translate(" + source.y + "," + source.x + ")"; + }) .remove(); - nodeExit.select("circle") - .attr("r", 1e-6); - nodeExit.select("text") - .style("fill-opacity", 1e-6); + nodeExit.select("circle").attr("r", 1e-6); + nodeExit.select("text").style("fill-opacity", 1e-6); // Update the links… - const link = svg.selectAll("path.link") - .data(links, d => { return d.target.id; }); + const link = svg.selectAll("path.link").data(links, d => { + return d.target.id; + }); // Enter any new links at the parent's previous position. - link.enter().insert("path", "g") + link.enter() + .insert("path", "g") .attr("class", "link") .attr("d", d => { - const o = {x: source.x0, y: source.y0}; - return diagonal({source: o, target: o}); + const o = { x: source.x0, y: source.y0 }; + return diagonal({ source: o, target: o }); }); // Transition links to their new position. link.transition() .duration(duration) .attr("d", diagonal); // Transition exiting nodes to the parent's new position. - link.exit().transition() + link.exit() + .transition() .duration(duration) .attr("d", d => { - const o = {x: source.x, y: source.y}; - return diagonal({source: o, target: o}); + const o = { x: source.x, y: source.y }; + return diagonal({ source: o, target: o }); }) .remove(); // Stash the old positions for transition. @@ -173,7 +213,7 @@ var ToolRecommendationView = Backbone.View.extend({ } update(d); if (d.tool_id !== undefined && d.tool_id !== "undefined" && d.tool_id !== null && d.tool_id !== "") { - document.location.href = `${getAppRoot()}` + 'tool_runner?tool_id=' + d.tool_id; + document.location.href = `${getAppRoot()}` + "tool_runner?tool_id=" + d.tool_id; } } function collapse(d) { diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js index 82c7dc82034..f973f10b288 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form.js +++ b/client/galaxy/scripts/mvc/tool/tool-form.js @@ -269,7 +269,7 @@ const View = Backbone.View.extend({ } this.$el.append(this._templateSuccess(response, job_def)); const enable_tool_recommendations = window.Galaxy.config.enable_tool_recommendations; - if (enable_tool_recommendations === true || enable_tool_recommendations === 'true') { + if (enable_tool_recommendations === true || enable_tool_recommendations === "true") { // show tool recommendations this.$el.append($("
", { id: "tool-recommendation-view" })); new ToolRecommendation.ToolRecommendationView({ From 8735e9ad0d218fd628982015a5874c5095a33c9f Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 26 Feb 2020 14:23:27 +0100 Subject: [PATCH 07/38] sort imports --- lib/galaxy/webapps/galaxy/api/workflows.py | 31 ++++++---------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 32186d785f6..d40fb741279 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -10,34 +10,19 @@ import os import requests import yaml -from galaxy import ( - exceptions, - model, - util -) -from galaxy.managers import ( - histories, - workflows -) +from galaxy import exceptions, model, util +from galaxy.managers import histories, workflows from galaxy.managers.jobs import fetch_job_states, invocation_job_source_iter from galaxy.model.item_attrs import UsesAnnotations -from galaxy.tool_shed.galaxy_install.install_manager import InstallRepositoryManager +from galaxy.tool_shed.galaxy_install.install_manager import \ + InstallRepositoryManager from galaxy.tools.parameters import populate_state from galaxy.tools.parameters.basic import workflow_building_modes from galaxy.util.sanitize_html import sanitize_html -from galaxy.web import ( - expose_api, - expose_api_anonymous_and_sessionless, - expose_api_raw, - format_return_as_json, -) -from galaxy.webapps.base.controller import ( - BaseAPIController, - SharableMixin, - url_for, - UsesStoredWorkflowMixin -) - +from galaxy.web import (expose_api, expose_api_anonymous_and_sessionless, + expose_api_raw, format_return_as_json) +from galaxy.webapps.base.controller import (BaseAPIController, SharableMixin, + UsesStoredWorkflowMixin, url_for) from galaxy.workflow.extract import extract_workflow from galaxy.workflow.modules import module_factory from galaxy.workflow.reports import generate_report From bb36a58179bea13ba4684f55fbb8d5b481431a9c Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 26 Feb 2020 15:29:09 +0100 Subject: [PATCH 08/38] Update import order --- lib/galaxy/webapps/galaxy/api/workflows.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index d40fb741279..af2f9a552df 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -10,6 +10,9 @@ import os import requests import yaml + +from gxformat2._yaml import ordered_dump + from galaxy import exceptions, model, util from galaxy.managers import histories, workflows from galaxy.managers.jobs import fetch_job_states, invocation_job_source_iter @@ -22,16 +25,16 @@ from galaxy.util.sanitize_html import sanitize_html from galaxy.web import (expose_api, expose_api_anonymous_and_sessionless, expose_api_raw, format_return_as_json) from galaxy.webapps.base.controller import (BaseAPIController, SharableMixin, - UsesStoredWorkflowMixin, url_for) + url_for, UsesStoredWorkflowMixin) from galaxy.workflow.extract import extract_workflow from galaxy.workflow.modules import module_factory from galaxy.workflow.reports import generate_report from galaxy.workflow.run import invoke, queue_invoke from galaxy.workflow.run_request import build_workflow_run_configs -from gxformat2._yaml import ordered_dump -from markupsafe import escape import h5py + +from markupsafe import escape import numpy as np from keras.models import model_from_json from sqlalchemy import desc, false, or_, true From 2b5efd7960974df0c3268152655b171777755948 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 27 Feb 2020 08:32:38 -0500 Subject: [PATCH 09/38] Fix import order in api/workflows. --- lib/galaxy/webapps/galaxy/api/workflows.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index af2f9a552df..a9cb1f469e2 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -8,17 +8,21 @@ import json import logging import os +import h5py +import numpy as np import requests import yaml - from gxformat2._yaml import ordered_dump +from keras.models import model_from_json +from markupsafe import escape +from sqlalchemy import desc, false, or_, true +from sqlalchemy.orm import joinedload from galaxy import exceptions, model, util from galaxy.managers import histories, workflows from galaxy.managers.jobs import fetch_job_states, invocation_job_source_iter from galaxy.model.item_attrs import UsesAnnotations -from galaxy.tool_shed.galaxy_install.install_manager import \ - InstallRepositoryManager +from galaxy.tool_shed.galaxy_install.install_manager import InstallRepositoryManager from galaxy.tools.parameters import populate_state from galaxy.tools.parameters.basic import workflow_building_modes from galaxy.util.sanitize_html import sanitize_html @@ -32,13 +36,6 @@ from galaxy.workflow.reports import generate_report from galaxy.workflow.run import invoke, queue_invoke from galaxy.workflow.run_request import build_workflow_run_configs -import h5py - -from markupsafe import escape -import numpy as np -from keras.models import model_from_json -from sqlalchemy import desc, false, or_, true -from sqlalchemy.orm import joinedload log = logging.getLogger(__name__) From 8340dda9206a2e14b333a923079f65d3e7034eaf Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Thu, 27 Feb 2020 22:32:45 +0100 Subject: [PATCH 10/38] restore imports to be on separate lines --- lib/galaxy/webapps/galaxy/api/workflows.py | 27 +++++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index a9cb1f469e2..c058bbd9eb6 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -18,18 +18,33 @@ from markupsafe import escape from sqlalchemy import desc, false, or_, true from sqlalchemy.orm import joinedload -from galaxy import exceptions, model, util -from galaxy.managers import histories, workflows +from galaxy import ( + exceptions, + model, + util +) +from galaxy.managers import ( + histories, + workflows +) from galaxy.managers.jobs import fetch_job_states, invocation_job_source_iter from galaxy.model.item_attrs import UsesAnnotations from galaxy.tool_shed.galaxy_install.install_manager import InstallRepositoryManager from galaxy.tools.parameters import populate_state from galaxy.tools.parameters.basic import workflow_building_modes from galaxy.util.sanitize_html import sanitize_html -from galaxy.web import (expose_api, expose_api_anonymous_and_sessionless, - expose_api_raw, format_return_as_json) -from galaxy.webapps.base.controller import (BaseAPIController, SharableMixin, - url_for, UsesStoredWorkflowMixin) +from galaxy.web import ( + expose_api, + expose_api_anonymous_and_sessionless, + expose_api_raw, + format_return_as_json, +) +from galaxy.webapps.base.controller import ( + BaseAPIController, + SharableMixin, + url_for, + UsesStoredWorkflowMixin +) from galaxy.workflow.extract import extract_workflow from galaxy.workflow.modules import module_factory from galaxy.workflow.reports import generate_report From 7c8ae9b85b32cec8a2d0d6b53388b228add6d67c Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Thu, 27 Feb 2020 22:33:53 +0100 Subject: [PATCH 11/38] R --- lib/galaxy/webapps/galaxy/api/workflows.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index c058bbd9eb6..d79fdae9b45 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -51,7 +51,6 @@ from galaxy.workflow.reports import generate_report from galaxy.workflow.run import invoke, queue_invoke from galaxy.workflow.run_request import build_workflow_run_configs - log = logging.getLogger(__name__) From 2f4f567e4b0fae68d0160d2e4f1a3a137540047d Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Thu, 27 Feb 2020 23:26:15 +0100 Subject: [PATCH 12/38] Add keras and tensorflow as conditional requirements --- lib/galaxy/dependencies/__init__.py | 6 ++++++ lib/galaxy/dependencies/conditional-requirements.txt | 4 ++++ .../dependencies/pipfiles/default/pinned-requirements.txt | 2 -- lib/galaxy/webapps/galaxy/api/workflows.py | 4 +++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 4d398f0f230..328e553cb73 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -194,6 +194,12 @@ class ConditionalDependencies(object): def check_influxdb(self): return 'influxdb' in self.error_report_modules + def check_keras(self): + return asbool(self.config["enable_tool_recommendations"]) + + def check_tensorflow(self): + return asbool(self.config["enable_tool_recommendations"]) + def optional(config_file=None): if not config_file: diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index b57b56bfcbb..23944172298 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -28,3 +28,7 @@ watchdog python-gitlab pygithub influxdb + +# Deep learning packages for tool recommendation +keras==2.2.4 +tensorflow==1.15.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index dc64eec6383..61ceec85d97 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -78,7 +78,6 @@ jmespath==0.9.4 jsonpatch==1.25 jsonpointer==2.0 jsonschema==3.2.0 -keras==2.2.4 keystoneauth1==3.18.0 kombu==4.6.7 lockfile==0.12.2 @@ -181,7 +180,6 @@ subprocess32==3.5.4 ; python_version < '3.0' svgwrite==1.3.1 tempita==0.5.2 tenacity==4.12.0 -tensorflow==1.15.0 typing-extensions==3.7.4.1 typing==3.7.4.1 ; python_version < '3.5' tzlocal==2.0.0 diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index d79fdae9b45..3a477020cbb 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -13,7 +13,6 @@ import numpy as np import requests import yaml from gxformat2._yaml import ordered_dump -from keras.models import model_from_json from markupsafe import escape from sqlalchemy import desc, false, or_, true from sqlalchemy.orm import joinedload @@ -675,6 +674,9 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn Create model and associated dictionaries for recommendations """ if not self.tool_recommendation_model_path: + # import moves from the top of file: in case the tool recommendation feature is disabled, + # keras is not downloaded because of conditional requirement and Galaxy does not build + from keras.models import model_from_json self.tool_recommendation_model_path = self.__download_model(remote_model_url) self.all_tools = dict() model_weights = list() From 5b15a8fa9999c869deb73ee3e431f8ac509b991d Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 4 Mar 2020 14:51:24 +0100 Subject: [PATCH 13/38] Fix failing test --- lib/galaxy/webapps/galaxy/api/workflows.py | 19 +++++++++++---- lib/galaxy_test/api/test_workflows.py | 27 +++++++++++++--------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 6ccfa3f3c1f..2fe44a58dc1 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -653,13 +653,17 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn remote_model_url = trans.app.config.tool_recommendation_model_path if 'tool_sequence' not in payload or remote_model_url is None: return - tool_sequence = payload.get('tool_sequence', "") + isModelSet = True + recommended_tools = dict() + tool_sequence = "" # collect tool recommendation preferences if set by admin self.__collect_admin_preferences(trans.app.config.admin_tool_recommendations_path) # recreate the neural network model to be used for prediction - self.__set_model(trans, remote_model_url) - # get the recommended tools for a tool sequence - recommended_tools = self.__compute_tool_prediction(trans, tool_sequence) + isModelSet = self.__set_model(trans, remote_model_url) + if isModelSet is True: + # get the recommended tools for a tool sequence + tool_sequence = payload.get('tool_sequence', "") + recommended_tools = self.__compute_tool_prediction(trans, tool_sequence) return { "current_tool": tool_sequence, "predicted_data": recommended_tools @@ -676,7 +680,11 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn if not self.tool_recommendation_model_path: # import moves from the top of file: in case the tool recommendation feature is disabled, # keras is not downloaded because of conditional requirement and Galaxy does not build - from keras.models import model_from_json + try: + from keras.models import model_from_json + except Exception as e: + trans.response.status = 400 + return False self.tool_recommendation_model_path = self.__download_model(remote_model_url) self.all_tools = dict() model_weights = list() @@ -711,6 +719,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn counter_layer_weights += 1 # set the model weights self.loaded_model.set_weights(model_weights) + return True def __collect_admin_preferences(self, admin_path): """ diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 5e957cec22c..45eb96e2646 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -337,17 +337,22 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): route = "workflows/get_tool_predictions" response = self._post(route, data=request) recommendation_response = response.json() - # check Ok response from the API - self._assert_status_code_is(response, 200) - # check the input tool sequence - assert recommendation_response["current_tool"] == request["tool_sequence"] - # check non-empty predictions list - predicted_tools = recommendation_response["predicted_data"]["children"] - assert len(predicted_tools) > 0 - # check for the correct predictions - for tool in predicted_tools: - assert tool["tool_id"] in actual_recommendations - break + is_empty = bool(recommendation_response["current_tool"]) + if is_empty is False: + self._assert_status_code_is(response, 400) + else: + # check Ok response from the API + self._assert_status_code_is(response, 200) + recommendation_response = response.json() + # check the input tool sequence + assert recommendation_response["current_tool"] == request["tool_sequence"] + # check non-empty predictions list + predicted_tools = recommendation_response["predicted_data"]["children"] + assert len(predicted_tools) > 0 + # check for the correct predictions + for tool in predicted_tools: + assert tool["tool_id"] in actual_recommendations + break def test_update(self): original_workflow = self.workflow_populator.load_workflow(name="test_import") From e69d70cad319ebc21cad27b892feeb035d886bdb Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 4 Mar 2020 15:18:57 +0100 Subject: [PATCH 14/38] Fix var --- lib/galaxy/webapps/galaxy/api/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 2fe44a58dc1..63ab76a7058 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -682,7 +682,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn # keras is not downloaded because of conditional requirement and Galaxy does not build try: from keras.models import model_from_json - except Exception as e: + except Exception: trans.response.status = 400 return False self.tool_recommendation_model_path = self.__download_model(remote_model_url) From 42abe1660e70717edaa3c4eb69dc9bbff7a15bae Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 10 Mar 2020 13:28:51 +0100 Subject: [PATCH 15/38] Migrate to VueJS from Backbone --- .../scripts/components/ToolRecommendation.vue | 286 ++++++++++++++++++ .../galaxy/scripts/mvc/tool-recommendation.js | 237 --------------- client/galaxy/scripts/mvc/tool/tool-form.js | 15 +- client/galaxy/style/scss/base.scss | 20 -- 4 files changed, 296 insertions(+), 262 deletions(-) create mode 100644 client/galaxy/scripts/components/ToolRecommendation.vue delete mode 100644 client/galaxy/scripts/mvc/tool-recommendation.js diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue new file mode 100644 index 00000000000..28886ab8bcd --- /dev/null +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -0,0 +1,286 @@ + + + + + diff --git a/client/galaxy/scripts/mvc/tool-recommendation.js b/client/galaxy/scripts/mvc/tool-recommendation.js deleted file mode 100644 index 09e9a29812f..00000000000 --- a/client/galaxy/scripts/mvc/tool-recommendation.js +++ /dev/null @@ -1,237 +0,0 @@ -import Backbone from "backbone"; -import Utils from "utils/utils"; -import * as d3 from "d3"; -import { getAppRoot } from "onload/loadConfig"; -import $ from "jquery"; - -var ToolRecommendationView = Backbone.View.extend({ - el: "#tool-recommendation-view", - - initialize: function(options) { - let toolId = options.toolId || ""; - const self = this; - if (toolId.indexOf("/") > 0) { - const toolIdSlash = toolId.split("/"); - toolId = toolIdSlash[toolIdSlash.length - 2]; - } - Utils.request({ - type: "POST", - url: `${getAppRoot()}api/workflows/get_tool_predictions`, - data: { tool_sequence: toolId }, - success: data => { - // get datatypes mapping - const datatypes_mapping = JSON.parse( - $.ajax({ - url: `${getAppRoot()}api/datatypes/mapping`, - async: false - }).responseText - ), - extToType = datatypes_mapping.ext_to_class_name, - typeToType = datatypes_mapping.class_to_classes, - predData = data.predicted_data; - if (data !== null && predData.children.length > 0) { - const filteredData = {}, - compatibleTools = {}, - filteredChildren = [], - outputDatatypes = predData["o_extensions"], - children = predData.children; - for (const nameObj of children.entries()) { - const inputDatatypes = nameObj[1]["i_extensions"]; - for (const out_t of outputDatatypes.entries()) { - for (const in_t of inputDatatypes.entries()) { - const child = extToType[out_t[1]], - parent = extToType[in_t[1]]; - if ( - (typeToType[child] && parent in typeToType[child]) === true || - out_t[1] === "input" || - out_t[1] === "_sniff_" || - out_t[1] === "input_collection" - ) { - compatibleTools[nameObj[1]["tool_id"]] = nameObj[1]["name"]; - break; - } - } - } - } - for (const id in compatibleTools) { - for (const nameObj of children.entries()) { - if (nameObj[1]["tool_id"] === id) { - filteredChildren.push(nameObj[1]); - break; - } - } - } - filteredData["o_extensions"] = predData["o_extensions"]; - filteredData["name"] = predData["name"]; - filteredData["children"] = filteredChildren; - if (filteredChildren.length > 0 && predData["is_deprecated"] === false) { - self.$el.append( - "
You have used " + - filteredData.name + - " tool. For further analysis, you could try using the following/recommended tools. The recommended tools are shown in the decreasing order of their scores predicted using machine learning analysis on workflows. A tool with a higher score (closer to 100%) may fit better as the following tool than a tool with a lower score. Please click on one of the following/recommended tools to open its definition.
" - ); - self.render_tree(filteredData); - } else if (predData["is_deprecated"] === true) { - self.$el.append( - "
You have used " + - predData.name + - " tool. " + - predData["message"] + - ".
" - ); - } - } - } - }); - }, - - render_tree: function(predicted_data) { - const margin = { top: 20, right: 30, bottom: 20, left: 250 }, - width = 900 - margin.right - margin.left, - height = 300 - margin.top - margin.bottom, - duration = 750; - const tree = d3.layout.tree().size([height, width]); - const diagonal = d3.svg.diagonal().projection(d => { - return [d.y, d.x]; - }); - const svg = d3 - .select("#tool-recommendation-view") - .append("svg") - .attr("width", width + margin.right + margin.left) - .attr("height", height + margin.top + margin.bottom) - .append("g") - .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - let i = 0, - root = null; - function update(source) { - // Compute the new tree layout. - const nodes = tree.nodes(root).reverse(), - links = tree.links(nodes); - // Normalize for fixed-depth. - nodes.forEach(d => { - d.y = d.depth * 180; - }); - // Update the nodes… - const node = svg.selectAll("g.node").data(nodes, d => { - return d.id || (d.id = ++i); - }); - // Enter any new nodes at the parent's previous position. - const nodeEnter = node - .enter() - .append("g") - .attr("class", "node") - .attr("transform", d => { - return "translate(" + source.y0 + "," + source.x0 + ")"; - }) - .on("click", click); - nodeEnter - .append("circle") - .attr("r", 1e-6) - .style("fill", d => { - return d._children ? "lightsteelblue" : "#fff"; - }); - nodeEnter - .append("text") - .attr("x", d => { - return d.children || d._children ? -10 : 10; - }) - .attr("dy", ".35em") - .attr("text-anchor", d => { - return d.children || d._children ? "end" : "start"; - }) - .text(d => { - return d.name; - }) - .style("fill-opacity", 1e-6); - nodeEnter.append("title").text(d => { - return d.children || d._children ? "Click to collapse" : "Click to open tool definition"; - }); - // Transition nodes to their new position. - const nodeUpdate = node - .transition() - .duration(duration) - .attr("transform", d => { - return "translate(" + d.y + "," + d.x + ")"; - }); - nodeUpdate - .select("circle") - .attr("r", 4.5) - .style("fill", d => { - return d._children ? "lightsteelblue" : "#fff"; - }); - nodeUpdate.select("text").style("fill-opacity", 1); - // Transition exiting nodes to the parent's new position. - const nodeExit = node - .exit() - .transition() - .duration(duration) - .attr("transform", d => { - return "translate(" + source.y + "," + source.x + ")"; - }) - .remove(); - nodeExit.select("circle").attr("r", 1e-6); - nodeExit.select("text").style("fill-opacity", 1e-6); - // Update the links… - const link = svg.selectAll("path.link").data(links, d => { - return d.target.id; - }); - // Enter any new links at the parent's previous position. - link.enter() - .insert("path", "g") - .attr("class", "link") - .attr("d", d => { - const o = { x: source.x0, y: source.y0 }; - return diagonal({ source: o, target: o }); - }); - // Transition links to their new position. - link.transition() - .duration(duration) - .attr("d", diagonal); - // Transition exiting nodes to the parent's new position. - link.exit() - .transition() - .duration(duration) - .attr("d", d => { - const o = { x: source.x, y: source.y }; - return diagonal({ source: o, target: o }); - }) - .remove(); - // Stash the old positions for transition. - nodes.forEach(d => { - d.x0 = d.x; - d.y0 = d.y; - }); - } - // Toggle children on click. - function click(d) { - if (d.children) { - d._children = d.children; - d.children = null; - } else { - d.children = d._children; - d._children = null; - } - update(d); - if (d.tool_id !== undefined && d.tool_id !== "undefined" && d.tool_id !== null && d.tool_id !== "") { - document.location.href = `${getAppRoot()}` + "tool_runner?tool_id=" + d.tool_id; - } - } - function collapse(d) { - if (d.children) { - d._children = d.children; - d._children.forEach(collapse); - d.children = null; - } - } - d3.select(self.frameElement).style("height", "400px"); - root = predicted_data; - root.x0 = height / 2; - root.y0 = 0; - root.children.forEach(collapse); - update(root); - } -}); - -export default { - ToolRecommendationView: ToolRecommendationView -}; diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js index f973f10b288..10386c03008 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form.js +++ b/client/galaxy/scripts/mvc/tool/tool-form.js @@ -13,7 +13,7 @@ import ToolFormBase from "mvc/tool/tool-form-base"; import Webhooks from "mvc/webhooks"; import Vue from "vue"; import ToolEntryPoints from "components/ToolEntryPoints/ToolEntryPoints"; -import ToolRecommendation from "mvc/tool-recommendation"; +import ToolRecommendation from "components/ToolRecommendation"; const View = Backbone.View.extend({ initialize: function(options) { @@ -271,10 +271,15 @@ const View = Backbone.View.extend({ const enable_tool_recommendations = window.Galaxy.config.enable_tool_recommendations; if (enable_tool_recommendations === true || enable_tool_recommendations === "true") { // show tool recommendations - this.$el.append($("
", { id: "tool-recommendation-view" })); - new ToolRecommendation.ToolRecommendationView({ - toolId: job_def.tool_id - }); + const ToolRecommendationInstance = Vue.extend(ToolRecommendation); + const vm = document.createElement("div"); + this.$el.append(vm); + const instance = new ToolRecommendationInstance({ + propsData: { + toolId: job_def.tool_id + } + }); + instance.$mount(vm); } this.$el.parent().scrollTop(0); // Show Webhook if job is running diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index c6b0e3f2e19..ed488b4f892 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1650,23 +1650,3 @@ body.reports { bottom: 0; background: url(../../images/largespinner.gif) no-repeat center center fixed; } - -/* For collapsible D3 tree */ - -.node { - cursor: pointer; -} -.node circle { - fill: #000; - stroke: #c2ebc2; - stroke-width: 0.3rem; -} -.node text { - font: 0.75rem sans-serif; -} - -.link { - fill: none; - stroke: #c2ebc2; - stroke-width: 0.3rem; -} From 2c723457086875854273c6bb69f697681ac53e1f Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 10 Mar 2020 15:03:46 +0100 Subject: [PATCH 16/38] Fix linting error --- client/galaxy/scripts/components/ToolRecommendation.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue index 28886ab8bcd..85c08679e5d 100644 --- a/client/galaxy/scripts/components/ToolRecommendation.vue +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -44,7 +44,7 @@ export default { methods: { loadRecommendations: function() { const self = this; - let toolId = this.getToolId; + const toolId = this.getToolId; Utils.request({ type: "POST", url: `${getAppRoot()}api/workflows/get_tool_predictions`, From b797b0fb4689e801b473e7b2d702175ecb18504e Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 10 Mar 2020 15:06:39 +0100 Subject: [PATCH 17/38] Fix indentation --- client/galaxy/scripts/mvc/tool/tool-form.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js index 10386c03008..212134fc247 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form.js +++ b/client/galaxy/scripts/mvc/tool/tool-form.js @@ -272,14 +272,14 @@ const View = Backbone.View.extend({ if (enable_tool_recommendations === true || enable_tool_recommendations === "true") { // show tool recommendations const ToolRecommendationInstance = Vue.extend(ToolRecommendation); - const vm = document.createElement("div"); - this.$el.append(vm); - const instance = new ToolRecommendationInstance({ - propsData: { - toolId: job_def.tool_id - } - }); - instance.$mount(vm); + const vm = document.createElement("div"); + this.$el.append(vm); + const instance = new ToolRecommendationInstance({ + propsData: { + toolId: job_def.tool_id + } + }); + instance.$mount(vm); } this.$el.parent().scrollTop(0); // Show Webhook if job is running From b6051585b82caeaf51f054fb4e96d5e339408f52 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 10 Mar 2020 16:42:46 +0100 Subject: [PATCH 18/38] Fix prettier --- .../scripts/components/ToolRecommendation.vue | 76 +++++++++---------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue index 85c08679e5d..6abcf09b319 100644 --- a/client/galaxy/scripts/components/ToolRecommendation.vue +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -1,10 +1,14 @@ @@ -48,31 +52,31 @@ export default { Utils.request({ type: "POST", url: `${getAppRoot()}api/workflows/get_tool_predictions`, - data: {tool_sequence: toolId}, + data: { tool_sequence: toolId }, success: data => { // get datatypes mapping const datatypes_mapping = JSON.parse( - $.ajax({ - url: `${getAppRoot()}api/datatypes/mapping`, - async: false - }).responseText - ), - extToType = datatypes_mapping.ext_to_class_name, - typeToType = datatypes_mapping.class_to_classes, - predData = data.predicted_data; + $.ajax({ + url: `${getAppRoot()}api/datatypes/mapping`, + async: false + }).responseText + ), + extToType = datatypes_mapping.ext_to_class_name, + typeToType = datatypes_mapping.class_to_classes, + predData = data.predicted_data; this.deprecated = predData["is_deprecated"]; if (data !== null && predData.children.length > 0) { const filteredData = {}, - compatibleTools = {}, - filteredChildren = [], - outputDatatypes = predData["o_extensions"], - children = predData.children; + compatibleTools = {}, + filteredChildren = [], + outputDatatypes = predData["o_extensions"], + children = predData.children; for (const nameObj of children.entries()) { const inputDatatypes = nameObj[1]["i_extensions"]; for (const out_t of outputDatatypes.entries()) { for (const in_t of inputDatatypes.entries()) { const child = extToType[out_t[1]], - parent = extToType[in_t[1]]; + parent = extToType[in_t[1]]; if ( (typeToType[child] && parent in typeToType[child]) === true || out_t[1] === "input" || @@ -98,8 +102,7 @@ export default { filteredData["children"] = filteredChildren; if (filteredChildren.length > 0 && this.deprecated === false) { self.renderD3Tree(filteredData); - } - else if (this.deprecated === true) { + } else if (this.deprecated === true) { this.deprecatedMessage = predData["message"]; } } @@ -108,9 +111,9 @@ export default { }, renderD3Tree: function(predictedTools) { const margin = { top: 20, right: 30, bottom: 20, left: 250 }, - width = 900 - margin.right - margin.left, - height = 300 - margin.top - margin.bottom, - duration = 750; + width = 900 - margin.right - margin.left, + height = 300 - margin.top - margin.bottom, + duration = 750; const tree = d3.layout.tree().size([height, width]); const diagonal = d3.svg.diagonal().projection(d => { return [d.y, d.x]; @@ -122,11 +125,11 @@ export default { .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); let i = 0, - root = null; + root = null; function update(source) { // Compute the new tree layout. const nodes = tree.nodes(root).reverse(), - links = tree.links(nodes); + links = tree.links(nodes); // Normalize for fixed-depth. nodes.forEach(d => { d.y = d.depth * 180; @@ -144,9 +147,7 @@ export default { return "translate(" + source.y0 + "," + source.x0 + ")"; }) .on("click", click); - nodeEnter - .append("circle") - .attr("r", 1e-6) + nodeEnter.append("circle").attr("r", 1e-6); nodeEnter .append("text") .attr("x", d => { @@ -170,11 +171,8 @@ export default { .attr("transform", d => { return "translate(" + d.y + "," + d.x + ")"; }); - nodeUpdate - .select("circle") - .attr("r", 4.5) - nodeUpdate.select("text") - .attr("class", "node-update") + nodeUpdate.select("circle").attr("r", 4.5); + nodeUpdate.select("text").attr("class", "node-update"); // Transition exiting nodes to the parent's new position. const nodeExit = node .exit() @@ -184,10 +182,8 @@ export default { return "translate(" + source.y + "," + source.x + ")"; }) .remove(); - nodeExit.select("circle") - .attr("r", 1e-6); - nodeExit.select("text") - .attr("class", "node-enter"); + nodeExit.select("circle").attr("r", 1e-6); + nodeExit.select("text").attr("class", "node-enter"); // Update the links const link = svg.selectAll("path.link").data(links, d => { return d.target.id; @@ -197,8 +193,8 @@ export default { .insert("path", "g") .attr("class", "link") .attr("d", d => { - const o = {x: source.x0, y: source.y0}; - return diagonal({source: o, target: o}); + const o = { x: source.x0, y: source.y0 }; + return diagonal({ source: o, target: o }); }); // Transition links to their new position. link.transition() From 82e6cd0874f6f43c1be8e0304849806f919b9db7 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Mon, 16 Mar 2020 12:57:14 +0100 Subject: [PATCH 19/38] Remove jquery and add axios --- .../scripts/components/ToolRecommendation.vue | 111 +++++++++--------- 1 file changed, 54 insertions(+), 57 deletions(-) diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue index 6abcf09b319..6d3dfbf5a19 100644 --- a/client/galaxy/scripts/components/ToolRecommendation.vue +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -17,7 +17,7 @@ import Utils from "utils/utils"; import * as d3 from "d3"; import { getAppRoot } from "onload/loadConfig"; -import $ from "jquery"; +import axios from "axios"; export default { props: { @@ -47,67 +47,64 @@ export default { }, methods: { loadRecommendations: function() { - const self = this; - const toolId = this.getToolId; - Utils.request({ - type: "POST", - url: `${getAppRoot()}api/workflows/get_tool_predictions`, - data: { tool_sequence: toolId }, - success: data => { - // get datatypes mapping - const datatypes_mapping = JSON.parse( - $.ajax({ - url: `${getAppRoot()}api/datatypes/mapping`, - async: false - }).responseText - ), - extToType = datatypes_mapping.ext_to_class_name, - typeToType = datatypes_mapping.class_to_classes, - predData = data.predicted_data; - this.deprecated = predData["is_deprecated"]; - if (data !== null && predData.children.length > 0) { - const filteredData = {}, - compatibleTools = {}, - filteredChildren = [], - outputDatatypes = predData["o_extensions"], - children = predData.children; - for (const nameObj of children.entries()) { - const inputDatatypes = nameObj[1]["i_extensions"]; - for (const out_t of outputDatatypes.entries()) { - for (const in_t of inputDatatypes.entries()) { - const child = extToType[out_t[1]], - parent = extToType[in_t[1]]; - if ( - (typeToType[child] && parent in typeToType[child]) === true || - out_t[1] === "input" || - out_t[1] === "_sniff_" || - out_t[1] === "input_collection" - ) { - compatibleTools[nameObj[1]["tool_id"]] = nameObj[1]["name"]; + const self = this, + toolId = this.getToolId, + url = getAppRoot() + "api/workflows/get_tool_predictions"; + axios + .post(url, { + tool_sequence: toolId + }) + .then(response => { + axios.get(getAppRoot() + "api/datatypes/mapping").then(responseMapping => { + const predData = response.data.predicted_data, + datatypesMapping = responseMapping.data, + extToType = datatypesMapping.ext_to_class_name, + typeToType = datatypesMapping.class_to_classes; + self.deprecated = predData.is_deprecated; + + if (response.data !== null && predData.children.length > 0) { + const filteredData = {}, + compatibleTools = {}, + filteredChildren = [], + outputDatatypes = predData.o_extensions, + children = predData.children; + for (const nameObj of children.entries()) { + const inputDatatypes = nameObj[1].i_extensions; + for (const out_t of outputDatatypes.entries()) { + for (const in_t of inputDatatypes.entries()) { + const child = extToType[out_t[1]], + parent = extToType[in_t[1]]; + if ( + (typeToType[child] && parent in typeToType[child]) === true || + out_t[1] === "input" || + out_t[1] === "_sniff_" || + out_t[1] === "input_collection" + ) { + compatibleTools[nameObj[1]["tool_id"]] = nameObj[1]["name"]; + break; + } + } + } + } + for (const id in compatibleTools) { + for (const nameObj of children.entries()) { + if (nameObj[1]["tool_id"] === id) { + filteredChildren.push(nameObj[1]); break; } } } - } - for (const id in compatibleTools) { - for (const nameObj of children.entries()) { - if (nameObj[1]["tool_id"] === id) { - filteredChildren.push(nameObj[1]); - break; - } + filteredData["o_extensions"] = predData.o_extensions; + filteredData["name"] = predData.name; + filteredData["children"] = filteredChildren; + if (filteredChildren.length > 0 && self.deprecated === false) { + self.renderD3Tree(filteredData); + } else if (self.deprecated === true) { + self.deprecatedMessage = predData.message; } } - filteredData["o_extensions"] = predData["o_extensions"]; - filteredData["name"] = predData["name"]; - filteredData["children"] = filteredChildren; - if (filteredChildren.length > 0 && this.deprecated === false) { - self.renderD3Tree(filteredData); - } else if (this.deprecated === true) { - this.deprecatedMessage = predData["message"]; - } - } - } - }); + }); + }); }, renderD3Tree: function(predictedTools) { const margin = { top: 20, right: 30, bottom: 20, left: 250 }, @@ -226,7 +223,7 @@ export default { } update(d); if (d.tool_id !== undefined && d.tool_id !== "undefined" && d.tool_id !== null && d.tool_id !== "") { - document.location.href = `${getAppRoot()}` + "tool_runner?tool_id=" + d.tool_id; + document.location.href = getAppRoot() + "tool_runner?tool_id=" + d.tool_id; } } function collapse(d) { From 6174c232a50371d4deae7c115cf6b5b556b56944 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Mon, 16 Mar 2020 12:58:26 +0100 Subject: [PATCH 20/38] Remove utils --- client/galaxy/scripts/components/ToolRecommendation.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue index 6d3dfbf5a19..88d0eaa4f4a 100644 --- a/client/galaxy/scripts/components/ToolRecommendation.vue +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -14,7 +14,6 @@ - - diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index ed488b4f892..c8ec6e2aee7 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1650,3 +1650,36 @@ body.reports { bottom: 0; background: url(../../images/largespinner.gif) no-repeat center center fixed; } + +/* Used for tree in tool recommendations */ +.node { + cursor: pointer; + circle { + fill: rgb(0, 0, 0); + stroke: rgb(194, 235, 194); + stroke-width: 0.3rem; + } + text { + font: 0.75rem sans-serif; + } +} + +.node-enter { + fill-opacity: 1e-6; +} + +.node-update { + fill-opacity: 1; +} + +.tree-size { + width: 100%; + height: 100%; + position: fixed; +} + +.link { + fill: none; + stroke: rgb(194, 235, 194); + stroke-width: 0.3rem; +} From db51c95c82f3e0c6bcc197e70e3587fd872fad33 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Mon, 16 Mar 2020 23:57:50 +0100 Subject: [PATCH 24/38] Fix import order --- lib/galaxy/webapps/galaxy/api/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index c2a055e2946..1dee03f7000 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -23,10 +23,10 @@ from galaxy.managers import ( histories, workflows ) -from galaxy.tools import tool_recommendations from galaxy.managers.jobs import fetch_job_states, invocation_job_source_iter from galaxy.model.item_attrs import UsesAnnotations from galaxy.tool_shed.galaxy_install.install_manager import InstallRepositoryManager +from galaxy.tools import tool_recommendations from galaxy.tools.parameters import populate_state from galaxy.tools.parameters.basic import workflow_building_modes from galaxy.util.sanitize_html import sanitize_html From 8d8d3d08f56587cdb728cfd2b7d0541a5d1c4275 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 17 Mar 2020 22:36:52 +0100 Subject: [PATCH 25/38] Update model path --- doc/source/admin/galaxy_options.rst | 5 ++--- lib/galaxy/config/sample/galaxy.yml.sample | 5 ++--- lib/galaxy/webapps/galaxy/config_schema.yml | 3 +-- lib/galaxy_test/api/test_workflows.py | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index e35ffb48af7..d1a68290537 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -4058,9 +4058,8 @@ :Description: Set remote path of the trained model (HDF5 file) for tool - recommendation. It is required to be set if - 'enable_tool_recommendations' property is enabled and set to true. -:Default: ``https://github.com/anuprulez/download_store/raw/tool_recommendation_model/tool_recommendation_model/tool_recommendation_model.hdf5`` + recommendation. +:Default: ``https://github.com/galaxyproject/galaxy-test-data/raw/master/tool_recommendation_model.hdf5`` :Type: str diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 3b39066b03f..43450b70e82 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1981,9 +1981,8 @@ galaxy: #enable_tool_recommendations: false # Set remote path of the trained model (HDF5 file) for tool - # recommendation. It is required to be set if - # 'enable_tool_recommendations' property is enabled and set to true. - #tool_recommendation_model_path: https://github.com/anuprulez/download_store/raw/tool_recommendation_model/tool_recommendation_model/tool_recommendation_model.hdf5 + # recommendation. + #tool_recommendation_model_path: https://github.com/galaxyproject/galaxy-test-data/raw/master/tool_recommendation_model.hdf5 # Set the number of predictions/recommendations to be made by the # model diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 66ed954360a..0180a2a1a5a 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2991,11 +2991,10 @@ mapping: tool_recommendation_model_path: type: str - default: 'https://github.com/anuprulez/download_store/raw/tool_recommendation_model/tool_recommendation_model/tool_recommendation_model.hdf5' + default: 'https://github.com/galaxyproject/galaxy-test-data/raw/master/tool_recommendation_model.hdf5' required: false desc: | Set remote path of the trained model (HDF5 file) for tool recommendation. - It is required to be set if 'enable_tool_recommendations' property is enabled and set to true. topk_recommendations: type: int diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 45eb96e2646..bb0a662896a 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -332,7 +332,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): return upload_response def test_get_tool_predictions(self): - request = {"tool_sequence": "Cut1", "remote_model_url": "https://github.com/anuprulez/download_store/raw/tool_recommendation_model/tool_recommendation_model/tool_recommendation_model.hdf5"} + request = {"tool_sequence": "Cut1", "remote_model_url": "https://github.com/galaxyproject/galaxy-test-data/raw/master/tool_recommendation_model.hdf5"} actual_recommendations = ['Filter1', 'cat1', 'addValue', 'comp1', 'Grep1'] route = "workflows/get_tool_predictions" response = self._post(route, data=request) From 26bfb80f4c885b0c55720a8354d664f297ff13c7 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 17 Mar 2020 23:10:37 +0100 Subject: [PATCH 26/38] ES6 strings for URL --- client/galaxy/scripts/components/ToolRecommendation.vue | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue index ff46367d641..c15c9774be3 100644 --- a/client/galaxy/scripts/components/ToolRecommendation.vue +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -50,13 +50,13 @@ export default { loadRecommendations: function() { const self = this, toolId = this.getToolId, - url = getAppRoot() + "api/workflows/get_tool_predictions"; + url = `${getAppRoot()}api/workflows/get_tool_predictions`; axios .post(url, { tool_sequence: toolId }) .then(response => { - axios.get(getAppRoot() + "api/datatypes/mapping").then(responseMapping => { + axios.get(`${getAppRoot()}api/datatypes/mapping`).then(responseMapping => { const predData = response.data.predicted_data, datatypesMapping = responseMapping.data, extToType = datatypesMapping.ext_to_class_name, @@ -222,8 +222,9 @@ export default { d._children = null; } update(d); - if (d.tool_id !== undefined && d.tool_id !== "undefined" && d.tool_id !== null && d.tool_id !== "") { - document.location.href = getAppRoot() + "tool_runner?tool_id=" + d.tool_id; + const tId = d.tool_id; + if (tId !== undefined && tId !== "undefined" && tId !== null && tId !== "") { + document.location.href = `${getAppRoot()}tool_runner?tool_id=${tId}`; } } function collapse(d) { From 70645f2a177e437a982fc0ee31fd213fd054b436 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 17 Mar 2020 23:26:18 +0100 Subject: [PATCH 27/38] Use theme colors --- client/galaxy/style/scss/base.scss | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index c8ec6e2aee7..9865e9e5e3b 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1655,8 +1655,8 @@ body.reports { .node { cursor: pointer; circle { - fill: rgb(0, 0, 0); - stroke: rgb(194, 235, 194); + fill: none; + stroke: lighten($brand-success, 20%); stroke-width: 0.3rem; } text { @@ -1680,6 +1680,6 @@ body.reports { .link { fill: none; - stroke: rgb(194, 235, 194); + stroke: lighten($brand-success, 20%); stroke-width: 0.3rem; } From 5133716e48d0a8f2cd56ff38090ce33cc74bb0c5 Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Tue, 17 Mar 2020 23:49:11 +0100 Subject: [PATCH 28/38] Remove margin vars --- .../scripts/components/ToolRecommendation.vue | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue index c15c9774be3..be74228e5f5 100644 --- a/client/galaxy/scripts/components/ToolRecommendation.vue +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -18,8 +18,6 @@ import * as d3 from "d3"; import { getAppRoot } from "onload/loadConfig"; import axios from "axios"; -const margin = { top: 20, right: 30, bottom: 20, left: 250 }; - export default { props: { toolId: { @@ -108,19 +106,19 @@ export default { }); }, renderD3Tree: function(predictedTools) { - const width = 900 - margin.right - margin.left, - height = 300 - margin.top - margin.bottom, - duration = 750; - const tree = d3.layout.tree().size([height, width]); - const diagonal = d3.svg.diagonal().projection(d => { - return [d.y, d.x]; - }); - const svg = d3 - .select("#tool-recommendation-view") - .append("svg") - .attr("class", "tree-size") - .append("g") - .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + const duration = 750, + x = 620, + y = 260, + tree = d3.layout.tree().size([y, x]), + diagonal = d3.svg.diagonal().projection(d => { + return [d.y, d.x]; + }), + svg = d3 + .select("#tool-recommendation-view") + .append("svg") + .attr("class", "tree-size") + .append("g") + .attr("transform", "translate(" + 250 + "," + 20 + ")"); let i = 0, root = null; function update(source) { @@ -235,7 +233,7 @@ export default { } } root = predictedTools; - root.x0 = height / 2; + root.x0 = y / 2; root.y0 = 0; root.children.forEach(collapse); update(root); From ae3f4629ca649dc6996f00e44cc6c74391eb0f4c Mon Sep 17 00:00:00 2001 From: Anup Kumar Date: Wed, 18 Mar 2020 00:36:48 +0100 Subject: [PATCH 29/38] Add header and brand-primary bg color --- client/galaxy/scripts/components/ToolRecommendation.vue | 1 + client/galaxy/style/scss/base.scss | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue index be74228e5f5..04891255862 100644 --- a/client/galaxy/scripts/components/ToolRecommendation.vue +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -1,6 +1,7 @@