From 516d7272291b4039ede60001bddf6e947e57a531 Mon Sep 17 00:00:00 2001 From: Tomithy Too Date: Sun, 26 Aug 2012 18:22:23 +0800 Subject: [PATCH 01/16] re-integrated phyloviz to a new fork of galaxy-central with a tat of minor touch up; save and edit now works --- lib/galaxy/datatypes/data.py | 44 +- lib/galaxy/datatypes/xml.py | 21 + lib/galaxy/visualization/phyloviz/__init__.py | 1 + .../visualization/phyloviz/baseparser.py | 125 +++ .../visualization/phyloviz/newickparser.py | 185 ++++ .../visualization/phyloviz/nexusparser.py | 107 ++ .../phyloviz/phyloviz_dataprovider.py | 35 + .../visualization/phyloviz/phyloxmlparser.py | 145 +++ lib/galaxy/web/controllers/phyloviz.py | 97 ++ lib/galaxy/web/controllers/visualization.py | 4 + static/scripts/viz/phyloviz.js | 955 ++++++++++++++++++ templates/root/history.mako | 11 + templates/root/history_common.mako | 11 + templates/visualization/phyloviz.mako | 320 ++++++ 14 files changed, 2060 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/visualization/phyloviz/__init__.py create mode 100644 lib/galaxy/visualization/phyloviz/baseparser.py create mode 100644 lib/galaxy/visualization/phyloviz/newickparser.py create mode 100644 lib/galaxy/visualization/phyloviz/nexusparser.py create mode 100644 lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py create mode 100644 lib/galaxy/visualization/phyloviz/phyloxmlparser.py create mode 100644 lib/galaxy/web/controllers/phyloviz.py create mode 100644 static/scripts/viz/phyloviz.js create mode 100644 templates/visualization/phyloviz.mako diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index f1d7cc3949c..2635bf0611c 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -719,7 +719,49 @@ class LineCount( Text ): pass class Newick( Text ): - pass + """New Hampshire/Newick Format""" + file_ext = "nhx" + + MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True ) + + def __init__(self, **kwd): + """Initialize foobar datatype""" + Text.__init__(self, **kwd) + + def init_meta( self, dataset, copy_from=None ): + Text.init_meta( self, dataset, copy_from=copy_from ) + + + def sniff( self, filename ): + """ Returning false as the newick format is too general and cannot be sniffed.""" + return False + + +class Nexus( Text ): + """Nexus format as used By Paup, Mr Bayes, etc""" + file_ext = "nex" + + MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True ) + + def __init__(self, **kwd): + """Initialize foobar datatype""" + Text.__init__(self, **kwd) + + def init_meta( self, dataset, copy_from=None ): + Text.init_meta( self, dataset, copy_from=copy_from ) + + + def sniff( self, filename ): + """All Nexus Files Simply puts a '#NEXUS' in its first line""" + f = open(filename, "r") + firstline = f.readline().upper() + f.close() + + if "#NEXUS" in firstline: + return True + else: + return False + # ------------- Utility methods -------------- diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index 37f34f55169..cb217c65177 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -76,3 +76,24 @@ class CisML( GenericXml ): dataset.blurb = 'file purged from disk' def sniff( self, filename ): return False + +class Phyloxml( GenericXml ): + """Format for defining phyloxml data http://www.phyloxml.org/""" + file_ext = "phyloxml" + def set_peek( self, dataset, is_multi_byte=False ): + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = 'Phyloxml data' + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def sniff( self, filename ): + """"Checking for keyword - 'phyloxml' always in lowercase in the first few lines""" + f = open(filename, "r") + firstlines = "".join(f.readlines(5)) + f.close() + if "phyloxml" in firstlines: + return True + return False \ No newline at end of file diff --git a/lib/galaxy/visualization/phyloviz/__init__.py b/lib/galaxy/visualization/phyloviz/__init__.py new file mode 100644 index 00000000000..a9a43536f7c --- /dev/null +++ b/lib/galaxy/visualization/phyloviz/__init__.py @@ -0,0 +1 @@ +__author__ = 'Tomithy' diff --git a/lib/galaxy/visualization/phyloviz/baseparser.py b/lib/galaxy/visualization/phyloviz/baseparser.py new file mode 100644 index 00000000000..f44d3606a17 --- /dev/null +++ b/lib/galaxy/visualization/phyloviz/baseparser.py @@ -0,0 +1,125 @@ +import json + +class Node(object): + """Node class of PhyloTree, which represents a CLAUDE in a phylogenetic tree""" + def __init__(self, nodeName, **kwargs): + """Creates a node and adds in the typical annotations""" + self.name, self.id = nodeName, kwargs.get("id", 0) + self.depth = kwargs.get("depth", 0) + self.children = [] + + self.isInternal = kwargs.get("isInternal", 0) + self.length, self.bootstrap = kwargs.get("length", 0), kwargs.get("bootstrap", None) + self.events = kwargs.get("events", "") + + # clean up boot strap values + if self.bootstrap == -1: + self.bootstrap = None + + def addChildNode(self, child): + """Adds a child node to the current node""" + if isinstance(child, Node): + self.children.append(child) + else: + self.children += child + + + def __str__(self): + return self.name + " id:" + str(self.id) + ", depth: " + str(self.depth) + + + def toJson(self): + """Converts the data in the node to a dict representation of json""" + thisJson = { + "name" : self.name, + "id" : self.id, + "depth" : self.depth, + "dist" : self.length + } + thisJson = self.addChildrenToJson(thisJson) + thisJson = self.addMiscToJson(thisJson) + return thisJson + + def addChildrenToJson(self, jsonDict): + """Needs a special method to addChildren, such that the key does not appear in the Jsondict when the children is empty + this requirement is due to the layout algorithm used by d3 layout for hiding subtree """ + if len(self.children) > 0: + children = [ node.toJson() for node in self.children] + jsonDict["children"] = children + return jsonDict + + + def addMiscToJson(self, jsonDict): + """Adds other misc attributes to json if they are present""" + if not self.events == "": + jsonDict["events"] = self.events + if not self.bootstrap == None: + jsonDict["bootstrap"] = self.bootstrap + return jsonDict + + + +class PhyloTree(object): + """Standardized python based class to represent the phylogenetic tree parsed from different + phylogenetic file formats.""" + + def __init__(self): + self.root, self.rootAttr = None, {} + self.nodes = {} + self.title = None + self.id = 1 + + def addAttributesToRoot(self, attrDict): + """Adds attributes to root, but first we put it in a temp store and bind it with root when .toJson is called""" + for key, value in attrDict.items(): + self.rootAttr[key] = value + + def makeNode(self, nodeName, **kwargs): + """Called to make a node within PhyloTree, arbitrary kwargs can be passed to annotate nodes + Tracks the number of nodes via internally incremented id""" + kwargs["id"] = self.id + self.id += 1 + return Node(nodeName, **kwargs) + + def addRoot(self, root): + """Creates a root for phyloTree""" + assert isinstance(root, Node) + root.parent = None + self.root = root + + def generateJsonableDict(self): + """Changes itself into a dictonary by recurssively calling the tojson on all its nodes. Think of it + as a dict in an array of dict in an array of dict and so on...""" + jsonTree = "" + if self.root: + assert isinstance(self.root, Node) + jsonTree = self.root.toJson() + for key, value in self.rootAttr.items(): + # transfer temporary stored attr to root + jsonTree[key] = value + else: + raise Exception("Root is not assigned!") + return jsonTree + + + +class Base_Parser(object): + """Base parsers contain all the methods to handle phylogeny tree creation and + converting the data to json that all parsers should have""" + + def __init__(self): + self.phyloTrees = [] + + def parseFile(self, filePath): + """Base method that all phylogeny file parser should have""" + raise Exception("Base method for phylogeny file parsers is not implemented") + + def toJson(self, jsonDict): + """Convenience method to get a json string from a python json dict""" + return json.dumps(jsonDict) + + def _writeJsonToFile(self, filepath, json): + """Writes the file out to the system""" + f = open(filepath, "w") + f.writelines(json) + f.close() diff --git a/lib/galaxy/visualization/phyloviz/newickparser.py b/lib/galaxy/visualization/phyloviz/newickparser.py new file mode 100644 index 00000000000..6e2576d4ef1 --- /dev/null +++ b/lib/galaxy/visualization/phyloviz/newickparser.py @@ -0,0 +1,185 @@ +from baseparser import Base_Parser, PhyloTree +import re + +class Newick_Parser(Base_Parser): + """For parsing trees stored in the newick format (.nhx) + It is necessarily more complex because this parser is later extended by Nexus for parsing newick as well..""" + + + def __init__(self): + super(Newick_Parser, self).__init__() + + + def parseFile(self, filePath): + """Parses a newick file to obtain the string inside. Returns: jsonableDict""" + with open(filePath, "r") as newickFile: + newickString = newickFile.read() + newickString = newickString.replace("\n", "").replace("\r", "") + return [self.parseData(newickString)], "Success" + + + def parseData(self, newickString): + """To be called on a newickString directly to parse it. Returns: jsonableDict""" + return self._parseNewickToJson(newickString) + + + def _parseNewickToJson(self, newickString, treeName=None, nameMap=None): + """parses a newick representation of a tree into a PhyloTree data structure, + which can be easily converted to json""" + self.phyloTree = PhyloTree() + newickString = self.cleanNewickString(newickString) + if nameMap: + newickString = self._mapName(newickString, nameMap) + + self.phyloTree.root = self.parseNode(newickString, 0) + if nameMap: + self.phyloTree.addAttributesToRoot({"treeName": treeName}) + + return self.phyloTree.generateJsonableDict() + + + def cleanNewickString(self, rawNewick): + """removing semi colon, and illegal json characters (\,',") and white spaces""" + return re.sub(r'\s|;|\"|\'|\\', '', rawNewick) + + + def _makeNodesFromString(self, string, depth): + """elements separated by comma could be empty""" + + if string.find("(") != -1: + raise Exception("Tree is not well form, location: " + string) + + childrenString = string.split(",") + childrenNodes = [] + + for childString in childrenString: + if len(childString) == 0: + continue + nodeInfo = childString.split(":") + name, length, bootstrap = "", None, -1 + if len(nodeInfo) == 2: # has length info + length = nodeInfo[1] + # checking for bootstap values + name = nodeInfo[0] + try: # Nexus may bootstrap in names position + name = float(name) + if 0<= name <= 1: + bootstrap = name + elif 1 <= name <= 100: + bootstrap = name / 100 + name = "" + except ValueError: + name = nodeInfo[0] + else: + name = nodeInfo[0] # string only contains name + node = self.phyloTree.makeNode(name, length=length, depth=depth, bootstrap= bootstrap) + childrenNodes += [node] + return childrenNodes + + + + def _mapName(self, newickString, nameMap): + """ + Necessary to replace names of terms inside nexus representation + Also, its here because Mailaud's doesnt deal with id_strings outside of quotes(" ") + """ + newString = "" + start = 0 + end = 0 + + for i in xrange(len(newickString)): + if newickString[i] == "(" or newickString[i] == ",": + if re.match(r"[,(]", newickString[i+1:]): + continue + else: + end = i + 1 + # i now refers to the starting position of the term to be replaced, + # we will next find j which is the ending pos of the term + for j in xrange(i+1, len(newickString)): + enclosingSymbol = newickString[j] # the immediate symbol after a common or left bracket which denotes the end of a term + if enclosingSymbol == ")" or enclosingSymbol == ":" or enclosingSymbol == ",": + termToReplace = newickString[end:j] + + newString += newickString[start : end] + nameMap[termToReplace] #+ "'" "'" + + start = j + break + + newString += newickString[start:] + return newString + + + def parseNode(self, string, depth): + """ Recursive method for parsing newick string, works by stripping down the string into substring + of newick contained with brackers, which is used to call itself. + Eg ... ( A, B, (D, E)C, F, G ) ... + We will make the preceeding nodes first A, B, then the internal node C, its children D, E, + and finally the succeeding nodes F, G""" + + # Base case where there is only an empty string + if string == "": + return + # Base case there its only an internal claude + if string.find("(") == -1: + return self._makeNodesFromString(string, depth) + + nodes, children = [], [] # nodes refer to the nodes on this level, children refers to the child of the + start = 0 + lenOfPreceedingInternalNodeString = 0 + bracketStack = [] + + for j in xrange(len(string)): + if string[j] == "(": #finding the positions of all the open brackets + bracketStack.append(j) + continue + if string[j] == ")": #finding the positions of all the closed brackets to extract claude + i = bracketStack.pop() + + if len(bracketStack) == 0: # is child of current node + + InternalNode = None + + #First flat call to make nodes of the same depth but from the preceeding string. + startSubstring = string[start + lenOfPreceedingInternalNodeString: i] + preceedingNodes = self._makeNodesFromString(startSubstring, depth) + nodes += preceedingNodes + + # Then We will try to see if the substring has any internal nodes first, make it then make nodes preceeding it and succeeding it. + if j + 1 < len(string): + stringRightOfBracket = string[j+1:] # Eg. '(b:0.4,a:0.3)c:0.3, stringRightOfBracket = c:0.3 + match = re.search(r"[\)\,\(]", stringRightOfBracket) + if match: + indexOfNextSymbol = match.start() + stringRepOfInternalNode = stringRightOfBracket[:indexOfNextSymbol] + internalNodes = self._makeNodesFromString( stringRepOfInternalNode, depth) + if len(internalNodes) > 0: + InternalNode = internalNodes[0] + lenOfPreceedingInternalNodeString = len(stringRepOfInternalNode) + else: # sometimes the node can be the last element of a string + InternalNode = self._makeNodesFromString(string[j+1:], depth)[0] + lenOfPreceedingInternalNodeString = len(string) - j + if InternalNode == None: #creating a generic node if it is unnamed + InternalNode = self.phyloTree.makeNode( "", depth=depth, isInternal=True ) #"internal-" + str(depth) + lenOfPreceedingInternalNodeString = 0 + + # recussive call to make the internal claude + childSubString = string[ i + 1 : j ] + InternalNode.addChildNode(self.parseNode(childSubString, depth + 1)) + + nodes.append(InternalNode) # we append the internal node later to preserve order + + start = j + 1 + continue + + if depth == 0: # if its the root node, we do nothing about it and return + return nodes[0] + + # Adding last most set of children + endString = string[start:] + if string[start-1] == ")": # if the symbol belongs to an internal node which is created previously, then we remove it from the string left to parse + match = re.search(r"[\)\,\(]", endString) + if match: + endOfNodeName = start + match.start() + 1 + endString = string[endOfNodeName:] + nodes += self._makeNodesFromString(endString, depth) + + return nodes diff --git a/lib/galaxy/visualization/phyloviz/nexusparser.py b/lib/galaxy/visualization/phyloviz/nexusparser.py new file mode 100644 index 00000000000..d703d168d97 --- /dev/null +++ b/lib/galaxy/visualization/phyloviz/nexusparser.py @@ -0,0 +1,107 @@ +from newickparser import Newick_Parser +import re + +MAX_READLINES = 200000 + + +class Nexus_Parser(Newick_Parser): + + def __init__(self): + super(Nexus_Parser, self).__init__() + + def parseFile(self, filePath): + """passes a file and extracts its Nexus content.""" + return self.parseNexus(filePath) + + + def parseNexus(self, filename): + """ Nexus data is stored in blocks between a line starting with begin and another line starting with end; + Commends inside square brackets are to be ignored, + For more information: http://wiki.christophchamp.com/index.php/NEXUS_file_format + Nexus can store multiple trees + """ + + with open( filename, "rt") as nex_file: + nexlines = nex_file.readlines() + + rowCount = 0 + inTreeBlock = False # sentinel to check if we are in a tree block + intranslateBlock = False # sentinel to check if we are in the translate region of the tree. Stores synonyms of the labellings + self.inCommentBlock = False + self.nameMapping = None # stores mapping representation used in nexus format + treeNames = [] + + for line in nexlines: + line = line.replace(";\n", "") + lline = line.lower() + + if rowCount > MAX_READLINES or (not nex_file) : + break + rowCount +=1 + # We are only interested in the tree block. + if "begin" in lline and "tree" in lline and not inTreeBlock: + inTreeBlock = True + continue + if inTreeBlock and "end" in lline[:3]: + inTreeBlock, currPhyloTree = False, None + continue + + if inTreeBlock: + + if "title" in lline: # Adding title to the tree + titleLoc = lline.find("title") + title = line[titleLoc + 5:].replace(" ", "") + + continue + + if "translate" in lline: + intranslateBlock = True + self.nameMapping = {} + continue + + if intranslateBlock: + mappingLine = self.splitLinebyWhitespaces(line) + key, value = mappingLine[1], mappingLine[2].replace(",", "").replace("'","") #replacing illegal json characters + self.nameMapping[key] = value + + # Extracting newick Trees + if "tree" in lline: + intranslateBlock = False + + treeLineCols = self.splitLinebyWhitespaces(line) + treeName, newick = treeLineCols[2], treeLineCols[-1] + + if newick == "": # Empty lines can be found in tree blocks + continue + + currPhyloTree = self._parseNewickToJson(newick, treeName, nameMap=self.nameMapping) + + self.phyloTrees.append(currPhyloTree) + treeIndex = len(self.phyloTrees) - 1 + treeNames.append( (treeName, treeIndex) ) # appending name of tree, and its index + continue + + return self.phyloTrees, treeNames + + + def splitLinebyWhitespaces(self, line): + """replace tabs and write spaces to a single write space, so we can properly split it.""" + return re.split(r"\s+", line) + + + def checkComments(self, line): + """Check to see if the line/lines is a comment.""" + if not self.inCommentBlock: + if "[" in line: + if "]" not in line: + self.inCommentBlock = True + else: + return "Nextline" # need to move on to the nextline after getting out of comment + else : + if "]" in line: + if line.rfind("[") > line.rfind("]"): + pass # a comment block is closed but another is open. + else: + self.inCommentBlock = False + return "Nextline" # need to move on to the nextline after getting out of comment + return "" \ No newline at end of file diff --git a/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py b/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py new file mode 100644 index 00000000000..026f33e51e6 --- /dev/null +++ b/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py @@ -0,0 +1,35 @@ +from newickparser import Newick_Parser +from nexusparser import Nexus_Parser +from phyloxmlparser import Phyloxml_Parser + +class Phyloviz_DataProvider(object): + + def __init__(self): + pass + + def parseFile(self, filepath, fileExt): + """returns [trees], meta + Trees are actually an array of JsonDicts. It's usually one tree, except in the case of Nexus + """ + jsonDicts, meta = [], {} + try: + if fileExt == "nhx": # parses newick files + newickParser = Newick_Parser() + jsonDicts, parseMsg = newickParser.parseFile(filepath) + elif fileExt == "phyloxml": # parses phyloXML files + phyloxmlParser = Phyloxml_Parser() + jsonDicts, parseMsg = phyloxmlParser.parseFile(filepath) + elif fileExt == "nex": # parses nexus files + nexusParser = Nexus_Parser() + jsonDicts, parseMsg = nexusParser.parseFile(filepath) + meta["trees"] = parseMsg + else: + raise Exception("File type is not supported") + + meta["msg"] = parseMsg + + except Exception: + jsonDicts, meta["msg"] = [], "Parse failed" + + return jsonDicts, meta + diff --git a/lib/galaxy/visualization/phyloviz/phyloxmlparser.py b/lib/galaxy/visualization/phyloviz/phyloxmlparser.py new file mode 100644 index 00000000000..ddfea8c30e0 --- /dev/null +++ b/lib/galaxy/visualization/phyloviz/phyloxmlparser.py @@ -0,0 +1,145 @@ +from baseparser import Base_Parser, PhyloTree, Node +from lxml import etree + +class Phyloxml_Parser(Base_Parser): + """Parses a phyloxml file into a json file that will be passed to PhyloViz for display""" + + def __init__(self): + super(Phyloxml_Parser, self).__init__() + self.phyloTree = PhyloTree() + self.tagsOfInterest = { + "clade": "", + "name" : "name", + "branch_length" : "length", + "confidence" : "bootstrap", + "events" : "events" + } + + def parseFile(self, filePath): + """passes a file and extracts its Phylogeny Tree content.""" + phyloXmlFile = open(filePath, "r") + + xmlTree = etree.parse(phyloXmlFile) + xmlRoot = xmlTree.getroot()[0] + self.nameSpaceIndex = xmlRoot.tag.rfind("}") + 1 # used later by the clean tag method to remove the name space in every element.tag + + phyloRoot = None + for child in xmlRoot: + childTag = self.cleanTag(child.tag) + if childTag == "clade": + phyloRoot = child + elif childTag == "name": + self.phyloTree.title = child.text + + self.phyloTree.root = self.parseNode(phyloRoot, 0) + jsonDict = self.phyloTree.generateJsonableDict() + return [jsonDict], "Success" + + + def parseNode(self, node, depth): + """Parses any node within a phyloxml tree and looks out for claude, which signals the creation of + nodes - internal OR leaf""" + assert isinstance(node, etree._Element) + + tag = self.cleanTag(node.tag) + if not tag == "clade": + return None + hasInnerClade = False + + # peeking once for parent and once for child to check if the node is internal + for child in node: + childTag = self.cleanTag(child.tag) + if childTag == "clade": + hasInnerClade = True + break + + if hasInnerClade: # this node is an internal node + currentNode = self._makeInternalNode(node, depth= depth) + for child in node: + child = self.parseNode(child, depth + 1) + if isinstance(child, Node): + currentNode.addChildNode(child) + + else: # this node is a leaf node + currentNode = self._makeLeafNode(node, depth=depth+1) + + return currentNode + + + def _makeLeafNode(self, leafNode, depth = 0 ): + """Makes leaf nodes by calling Phylotree methods""" + node = {} + for child in leafNode: + childTag = self.cleanTag(child.tag) + if childTag in self.tagsOfInterest: + key = self.tagsOfInterest[childTag] # need to map phyloxml terms to ours + node[key] = child.text + + node["depth"] = depth + return self.phyloTree.makeNode(self._getNodeName(leafNode), **node) + + def _getNodeName(self, node, depth=-1): + """Gets the name of a claude. It handles the case where a taxonomy node is involved""" + + def getTagFromTaxonomyNode(node): + """Returns the name of a taxonomy node. A taxonomy node have to be treated differently as the name + is embedded one level deeper""" + phyloxmlTaxoNames = { + "common_name" : "", + "scientific_name" : "", + "code" : "" + } + for child in node: + childTag = self.cleanTag(child.tag) + if childTag in phyloxmlTaxoNames: + return child.text + return "" + + nodeName = "" + for child in node: + childTag = self.cleanTag(child.tag) + if childTag == "name" : + nodeName = child.text + break + elif childTag == "taxonomy": + nodeName = getTagFromTaxonomyNode(child) + break + + return nodeName + + + def _makeInternalNode(self, internalNode, depth=0): + """ Makes an internal node from an element object that is guranteed to be a parent node. + Gets the value of interests like events and appends it to a custom node object that will be passed to PhyloTree to make nodes + """ + node = {} + for child in internalNode: + childTag = self.cleanTag(child.tag) + if childTag == "clade": + continue + elif childTag in self.tagsOfInterest: + if childTag == "events": # events is nested 1 more level deeper than others + key, text = "events", self.cleanTag(child[0].tag) + else: + key = self.tagsOfInterest[childTag] + text = child.text + node[key] = text + + + return self.phyloTree.makeNode(self._getNodeName(internalNode, depth), **node) + + + def cleanTag(self, tagString): + return tagString[self.nameSpaceIndex:] + + +if __name__=="__main__": + + # Files tested against + parser = Phyloxml_Parser() + filepath = "../data/" +"apaf.xml" + # filepath = "../data/" +"12_multiple_supports.xml" + + # filepath = "../data/" +"bcl_2.xml" + # filepath = "../data/" +"reducedXml.xml" + parser.parseFile(filepath) diff --git a/lib/galaxy/web/controllers/phyloviz.py b/lib/galaxy/web/controllers/phyloviz.py new file mode 100644 index 00000000000..a2f0356f89c --- /dev/null +++ b/lib/galaxy/web/controllers/phyloviz.py @@ -0,0 +1,97 @@ +import pkg_resources +pkg_resources.require( "bx-python" ) + +from galaxy.util.json import to_json_string, from_json_string +from galaxy.web.base.controller import * +from galaxy.visualization.phyloviz.phyloviz_dataprovider import Phyloviz_DataProvider + + +class PhyloVizController( BaseUIController, UsesVisualizationMixin, UsesHistoryDatasetAssociationMixin, SharableMixin ): + """ + Controller for phyloViz browser interface. + """ + def __init__(self, app ): + BaseUIController.__init__( self, app ) + + @web.expose + @web.require_login() + def index( self, trans, dataset_id = None, **kwargs ): + """ + The index method is called using phyloviz/ with a dataset id passed in. + The relevant data set is then retrieved via get_json_from_datasetId which interfaces with the parser + The json representation of the phylogenetic tree along with the config is then written in the .mako template and passed back to the user + """ + json, config = self.get_json_from_datasetId(trans, dataset_id) + config["saved_visualization"] = False + return trans.fill_template( "visualization/phyloviz.mako", data = json, config=config) + + + @web.expose + def visualization(self, trans, id): + """ + Called using a viz_id (id) to retrieved stored visualization data (in json format) and all the viz_config + """ + viz = self.get_visualization(trans, id) + config = self.get_visualization_config(trans, viz) + config["saved_visualization"] = True + data = config["root"] + + return trans.fill_template( "visualization/phyloviz.mako", data = data, config=config) + + + @web.expose + @web.json + def load_visualization_json(self, trans, viz_id): + """ + Though not used in current implementation, this provides user with a convenient method to retrieve the viz_data & viz_config via json. + """ + viz = self.get_visualization(trans, viz_id) + viz_config = self.get_visualization_config(trans, viz) + viz_config["saved_visualization"] = True + return { + "data" : viz_config["root"], + "config" : viz_config + } + + + @web.expose + @web.json + def getJsonData(self, trans, dataset_id, treeIndex=0): + """ + Method to retrieve data asynchronously via json format. Retriving from here rather than + making a direct datasets/ call allows for some processing and event capturing + """ + treeIndex = int(treeIndex) + json, config = self.get_json_from_datasetId(trans, dataset_id, treeIndex) + packedJson = { + "data" : json, + "config" : config + } + + return packedJson + + + def get_json_from_datasetId(self, trans, dataset_id, treeIndex=0): + """ + For interfacing phyloviz controllers with phyloviz visualization data provider (parsers) + """ + dataset = self.get_dataset(trans, dataset_id) + fileExt, filepath = dataset.ext, dataset.file_name # .name stores the name of the dataset from the orginal upload + json, config = "", {} # config contains properties of the tree and file + + if fileExt == "json": + something, json = self.get_data(dataset) + else: + try: + pd = Phyloviz_DataProvider() + json, config = pd.parseFile(filepath, fileExt) + json = json[treeIndex] + except Exception: + pass + + config["title"] = dataset.display_name() + config["ext"] = fileExt + config["dataset_id"] = dataset_id + config["treeIndex"] = treeIndex + + return json, config diff --git a/lib/galaxy/web/controllers/visualization.py b/lib/galaxy/web/controllers/visualization.py index 4f93a37fda5..c6d63d1c9e3 100644 --- a/lib/galaxy/web/controllers/visualization.py +++ b/lib/galaxy/web/controllers/visualization.py @@ -16,6 +16,10 @@ class VisualizationListGrid( grids.Grid ): action = "paramamonster" elif item.type == "circster": action = "circster" + elif item.type == "phyloviz": + # Support phyloviz + controller = "phyloviz" + action = "visualization" return dict( controller=controller, action=action, id=item.id ) # Grid definition diff --git a/static/scripts/viz/phyloviz.js b/static/scripts/viz/phyloviz.js new file mode 100644 index 00000000000..73b2f9aca87 --- /dev/null +++ b/static/scripts/viz/phyloviz.js @@ -0,0 +1,955 @@ +var UserMenuBase = Backbone.View.extend({ + /** + * Base class of any menus that takes in user interaction. Contains checking methods. + */ + + className: 'UserMenuBase', + + isAcceptableValue : function ($inputKey, min, max) { + /** + * Check if an input value is a number and falls within max min. + */ + var self = this, + value = $inputKey.val(), + fieldName = $inputKey.attr("displayLabel") || $inputKey.attr("id").replace("phyloViz", ""); + + function isNumeric(n) { + return !isNaN(parseFloat(n)) && isFinite(n); + } + + if (!isNumeric(value)){ + alert(fieldName + " is not a number!"); + return false; + } + + if ( value > max){ + alert(fieldName + " is too large."); + return false; + } else if ( value < min) { + alert(fieldName + " is too small."); + return false; + } + return true; + }, + + hasIllegalJsonCharacters : function($inputKey) { + /** + * Check if any user string inputs has illegal characters that json cannot accept + */ + if ($inputKey.val().search(/"|'|\\/) !== -1){ + alert("Named fields cannot contain these illegal characters: double quote(\"), single guote(\'), or back slash(\\). "); + return true; + } + return false; + } +}); + + +function PhyloTreeLayout() { + /** + * -- Custom Layout call for phyloViz to suit the needs of a phylogenetic tree. + * -- Specifically: 1) Nodes have a display display of (= evo dist X depth separation) from their parent + * 2) Nodes must appear in other after they have expand and contracted + */ + + var self = this, + hierarchy = d3.layout.hierarchy().sort(null).value(null), + height = 360, // ! represents both the layout angle and the height of the layout, in px + layoutMode = "Linear", + leafHeight = 18, // height of each individual leaf node + depthSeparation = 200, // separation between nodes of different depth, in px + leafIndex = 0, // change to recurssive call + defaultDist = 0.5, // tree defaults to 0.5 dist if no dist is specified + maxTextWidth = 50; // maximum length of the text labels + + + self.leafHeight = function(inputLeafHeight){ + if (typeof inputLeafHeight === "undefined"){ return leafHeight; } + else { leafHeight = inputLeafHeight; return self;} + }; + + self.layoutMode = function(mode){ + if (typeof mode === "undefined"){ return layoutMode; } + else { layoutMode = mode; return self;} + }; + + self.layoutAngle = function(angle) { // changes the layout angle of the display, which is really changing the height + if (typeof angle === "undefined"){ return height; } + if (isNaN(angle) || angle < 0 || angle > 360) { return self; } // to use default if the user puts in strange values + else { height = angle; return self;} + }; + + self.separation = function(dist){ // changes the dist between the nodes of different depth + if (typeof dist === "undefined"){ return depthSeparation; } + else { depthSeparation = dist; return self;} + }; + + self.links = function (nodes) { // uses d3 native method to generate links. Done. + return d3.layout.tree().links(nodes); + }; + + // -- Custom method for laying out phylogeny tree in a linear fashion + self.nodes = function (d, i) { + var _nodes = hierarchy.call(self, d, i), // self is to find the depth of all the nodes, assumes root is passed in + nodes = [], + maxDepth = 0, + numLeaves = 0; + + // changing from hierarchy's custom format for data to usable format + _nodes.forEach(function (_node){ + var node = _node.data; + node.depth = _node.depth; + maxDepth = node.depth > maxDepth ? node.depth : maxDepth; //finding max depth of tree + nodes.push(node); + }); + // counting the number of leaf nodes and assigning max depth to nodes that do not have children to flush all the leave nodes + nodes.forEach(function(node){ + if ( !node.children ) { //&& !node._children + numLeaves += 1; + node.depth = maxDepth; // if a leaf has no child it would be assigned max depth + } + }); + + leafHeight = layoutMode === "Circular" ? height / numLeaves : leafHeight; + leafIndex = 0; + layout(nodes[0], maxDepth, leafHeight, null); + + return nodes; + }; + + + function layout (node, maxDepth, vertSeparation, parent) { + /** + * -- Function with side effect of adding x0, y0 to all child; take in the root as starting point + * assuming that the leave nodes would be sorted in presented order + * horizontal(y0) is calculated according to (= evo dist X depth separation) from their parent + * vertical (x0) - if leave node: find its order in all of the leave node === node.id, then multiply by verticalSeparation + * - if parent node: is place in the mid point all of its children nodes + * -- The layout will first calculate the y0 field going towards the leaves, and x0 when returning + */ + var children = node.children, + sumChildVertSeparation = 0; + + // calculation of node's dist from parents, going down. + var dist = node.dist || defaultDist; + dist = dist > 1 ? 1 : dist; // We constrain all dist to be less than one + node.dist = dist; + if (parent !== null){ + node.y0 = parent.y0 + dist * depthSeparation; + } else { //root node + node.y0 = maxTextWidth; + } + + + // if a node have no children, we will treat it as a leaf and start laying it out first + if (!children) { + node.x0 = leafIndex++ * vertSeparation; + } else { + // if it has children, we will visit all its children and calculate its position from its children + children.forEach( function (child) { + child.parent = node; + sumChildVertSeparation += layout(child, maxDepth, vertSeparation, node); + }); + node.x0 = sumChildVertSeparation / children.length; + } + + // adding properties to the newly created node + node.x = node.x0; + node.y = node.y0; + return node.x0; + } + return self; +} + + +/** + * -- PhyloTree Model -- + */ +var PhyloTree = Visualization.extend({ + defaults : { + layout: "Linear", + separation : 250, // px dist between nodes of different depth to represent 1 evolutionary until + leafHeight: 18, + type : "phyloviz", // visualization type + title : "Title", + scaleFactor: 1, + translate: [0,0], + fontSize: 12, //fontSize of node label + selectedNode : null, + nodeAttrChangedTime : 0 + }, + + root : {}, // Root has to be its own independent object because it is not part of the viz_config + + toggle : function (d) { + /** + * Mechanism to expand or contract a single node. Expanded nodes have a children list, while for + * contracted nodes the list is stored in _children. Nodes with their children data stored in _children will not have their + * children rendered. + */ + if(typeof d === "undefined") {return ;} + if (d.children ) { + d._children = d.children; + d.children = null; + } else { + d.children = d._children; + d._children = null; + } + }, + + toggleAll : function(d) { + /** + * Contracts the phylotree to a single node by repeatedly calling itself to place all the list + * of children under _children. + */ + if (d.children && d.children.length !== 0) { + d.children.forEach(this.toggleAll); + toggle(d); + } + }, + + getData : function (){ + /** + * Return the data of the tree. Used for preserving state. + */ + return this.root; + }, + + save: function() { + /** + * Overriding the default save mechanism to do some clean of circular reference of the + * phyloTree and to include phyloTree in the saved json + */ + var root = this.root; + cleanTree(root); + this.set("root", root); + + function cleanTree(node){ + // we need to remove parent to delete circular reference + delete node.parent; + + // removing unnecessary attributes + if (node._selected){ delete node._selected;} + + node.children ? node.children.forEach(cleanTree) : 0; + node._children ? node._children.forEach(cleanTree) : 0; + } + + var config = jQuery.extend(true, {}, this.attributes); + config["selectedNode"] = null; + + show_message("Saving to Galaxy", "progress"); + + return $.ajax({ + url: this.url(), + type: "POST", + dataType: "json", + data: { + vis_json: JSON.stringify(config) + }, + success: function(res){ + var viz_id = res.url.split("id=")[1].split("&")[0], + viz_url = "/phyloviz/visualization?id=" + viz_id; + window.history.pushState({}, "", viz_url + window.location.hash); + hide_modal(); + } + }); + } +}); + + + +/** + * -- Views -- + */ +var PhylovizLayoutBase = Backbone.View.extend({ + /** + * Stores the default variable for setting up the visualization + */ + defaults : { + nodeRadius : 4.5 // radius of each node in the diagram + }, + + + stdInit : function (options) { + /** + * Common initialization in layouts + */ + + var self = this; + self.model.on("change:separation change:leafHeight change:fontSize change:nodeAttrChangedTime", self.updateAndRender, self); + + self.vis = options.vis; + self.i = 0; + self.maxDepth = -1; // stores the max depth of the tree + + self.width = options.width; + self.height = options.height; + }, + + + updateAndRender : function(source) { + /** + * Updates the visualization whenever there are changes in the expansion and contraction of nodes + * AND possibly when the tree is edited. + */ + var vis = d3.select(".vis"), + self = this; + source = source || self.model.root; + + self.renderNodes(source); + self.renderLinks(source); + self.addTooltips(); + }, + + + renderLinks : function(source) { + /** + * Renders the links for the visualization. + */ + var self = this; + var diagonal = self.diagonal; + var duration = self.duration; + var layoutMode = self.layoutMode; + var link = self.vis.selectAll("g.completeLink") + .data(self.tree.links(self.nodes), function(d) { return d.target.id; }); + + var calcalateLinePos = function(d) { + d.pos0 = d.source.y0 + " " + d.source.x0; // position of the source node <=> starting location of the line drawn + d.pos1 = d.source.y0 + " " + d.target.x0; // position where the line makes a right angle bend + d.pos2 = d.target.y0 + " " + d.target.x0; // point where the horizontal line becomes a dotted line + }; + + var linkEnter = link.enter().insert("svg:g","g.node") + .attr("class", "completeLink"); + + + linkEnter.append("svg:path") + .attr("class", "link") + .attr("d", function(d) { + calcalateLinePos(d); + return "M " + d.pos0 + " L " + d.pos1; + }); + + var linkUpdate = link.transition().duration(500); + + linkUpdate.select("path.link") + .attr("d", function(d) { + calcalateLinePos(d); + return "M " + d.pos0 + " L " + d.pos1 + " L " + d.pos2; + }); + + var linkExit = link.exit().remove(); + + }, + + // User Interaction methods below + + selectNode : function(node){ + /** + * Displays the information for editting + */ + var self = this; + d3.selectAll("g.node") + .classed("selectedHighlight", function(d){ + if (node.id === d.id){ + if(node._selected) { // for de=selecting node. + delete node._selected; + return false; + } else { + node._selected = true; + return true; + } + } + return false; + }); + + self.model.set("selectedNode", node); + $("#phyloVizSelectedNodeName").val(node.name); + $("#phyloVizSelectedNodeDist").val(node.dist); + $("#phyloVizSelectedNodeAnnotation").val(node.annotation || ""); + }, + + addTooltips : function (){ + /** + * Creates bootstrap tooltip for the visualization. Has to be called repeatedly due to newly generated + * enterNodes + */ + $(".bs-tooltip").remove(); //clean up tooltip, just in case its listeners are removed by d3 + $(".node") + .attr("data-original-title", function(){ + var d = this.__data__, + annotation = d.annotation || "None" ; + return d ? (d.name ? d.name + "
" : "") + "Dist: " + d.dist + "
Annotation: " + annotation: ""; + }) + .tooltip({'placement':'top', 'trigger' : 'hover'}); + + } +}); + + + + +var PhylovizLinearView = PhylovizLayoutBase.extend({ + /** + * Linea layout class of Phyloviz, is responsible for rendering the nodes + * calls PhyloTreeLayout to determine the positions of the nodes + */ + initialize : function(options){ + // Default values of linear layout + var self = this; + self.margins = options.margins; + self.layoutMode = "Linear"; + + self.stdInit(options); + + self.layout(); + self.updateAndRender(self.model.root); + }, + + layout : function() { + /** + * Creates the basic layout of a linear tree by precalculating fixed values. + * One of calculations are also made here + */ + + var self = this; + + self.tree = new PhyloTreeLayout().layoutMode("Linear"); + self.diagonal = d3.svg.diagonal() + .projection(function(d) { return [d.y, d.x ]; }); + }, + + renderNodes : function (source) { + /** + * Renders the nodes base on Linear layout. + */ + var self = this, + fontSize = self.model.get("fontSize") + "px"; + + // assigning properties from models + self.tree.separation(self.model.get("separation")).leafHeight(self.model.get("leafHeight")); + + var duration = 500, + nodes = self.tree.separation(self.model.get("separation")).nodes(self.model.root); + + var node = self.vis.selectAll("g.node") + .data(nodes, function(d) { return d.name + d.id || (d.id = ++self.i); }); + + // These variables has to be passed into update links which are in the base methods + self.nodes = nodes; + self.duration = duration; + + // ------- D3 ENTRY -------- + // Enter any new nodes at the parent's previous position. + var nodeEnter = node.enter().append("svg:g") + .attr("class", "node") + .on("dblclick", function(){ d3.event.stopPropagation(); }) + .on("click", function(d) { + if (d3.event.altKey) { + self.selectNode(d); // display info if alt is pressed + } else { + if(d.children && d.children.length === 0){ return;} // there is no need to toggle leaves + self.model.toggle(d); // contract/expand nodes at data level + self.updateAndRender(d); // re-render the tree + } + }); + + nodeEnter.attr("transform", function(d) { return "translate(" + source.y0 + "," + source.x0 + ")"; }); + + nodeEnter.append("svg:circle") + .attr("r", 1e-6) + .style("fill", function(d) { return d._children ? "lightsteelblue" : "#fff"; }); + + nodeEnter.append("svg:text") + .attr("class", "nodeLabel") + .attr("x", function(d) { return d.children || d._children ? -10 : 10; }) + .attr("dy", ".35em") + .attr("text-anchor", function(d) { return d.children || d._children ? "end" : "start"; }) + .style("fill-opacity", 1e-6); + + // ------- D3 TRANSITION -------- + // Transition nodes to their new position. + var nodeUpdate = node.transition() + .duration(duration); + + nodeUpdate.attr("transform", function(d) { + return "translate(" + d.y + "," + d.x + ")"; }); + + nodeUpdate.select("circle") + .attr("r", self.defaults.nodeRadius) + .style("fill", function(d) { return d._children ? "lightsteelblue" : "#fff"; }); + + nodeUpdate.select("text") + .style("fill-opacity", 1) + .style("font-size", fontSize) + .text(function(d) { return d.name; }); + + // ------- D3 EXIT -------- + // Transition exiting nodes to the parent's new position. + var nodeExit =node.exit().transition() + .duration(duration) + .remove(); + + nodeExit.select("circle") + .attr("r", 1e-6); + + nodeExit.select("text") + .style("fill-opacity", 1e-6); + + // Stash the old positions for transition. + nodes.forEach(function(d) { + d.x0 = d.x; // we need the x0, y0 for parents with children + d.y0 = d.y; + }); + } + +}); + +var PhylovizView = Backbone.View.extend({ + + className: 'phyloviz', + + initialize: function(options) { + var self = this; + // -- Default values of the vis + self.MIN_SCALE = 0.05; //for zooming + self.MAX_SCALE = 5; + self.MAX_DISPLACEMENT = 500; + self.margins = [10, 60, 10, 80]; + + self.width = $("#PhyloViz").width(); + self.height = $("#PhyloViz").height(); + self.radius = self.width; + self.data = options.data; + + // -- Events Phyloviz view responses to + $(window).resize(function(){ + self.width = $("#PhyloViz").width(); + self.height = $("#PhyloViz").height(); + self.render(); + }); + + // -- Create phyloTree model + self.phyloTree = new PhyloTree(options.config); + self.phyloTree.root = self.data; + + // -- Set up UI functions of main view + self.zoomFunc = d3.behavior.zoom().scaleExtent([self.MIN_SCALE, self.MAX_SCALE]); + self.zoomFunc.translate(self.phyloTree.get("translate")); + self.zoomFunc.scale(self.phyloTree.get("scaleFactor")); + + // -- set up header buttons, search and settings menu + self.navMenu = new HeaderButtons(self); + self.settingsMenu = new SettingsMenu({phyloTree : self.phyloTree}); + self.nodeSelectionView = new NodeSelectionView({phyloTree : self.phyloTree}); + self.search = new PhyloVizSearch(); + + + setTimeout(function(){ // using settimeout to call the zoomAndPan function according to the stored attributes in viz_config + self.zoomAndPan(); + }, 1000); + }, + + render: function(){ + // -- Creating helper function for vis. -- + var self = this; + $("#PhyloViz").empty(); + + // -- Layout viz. -- + self.mainSVG = d3.select("#PhyloViz").append("svg:svg") + .attr("width", self.width) + .attr("height", self.height) + .attr("pointer-events", "all") + .call(self.zoomFunc.on("zoom", function(){ + self.zoomAndPan(); + })); + + self.boundingRect = self.mainSVG.append("svg:rect") + .attr("class", "boundingRect") + .attr("width", self.width) + .attr("height", self.height) + .attr("stroke", "black") + .attr("fill", "white"); + + self.vis = self.mainSVG + .append("svg:g") + .attr("class", "vis"); + + self.layoutOptions = { + model : self.phyloTree, + width : self.width, + height : self.height, + vis: self.vis, + margins: self.margins + }; + + // -- Creating Title + $("#title").text("Phylogenetic Tree from " + self.phyloTree.get("title") + ":"); + + // -- Create Linear view instance -- + var linearView = new PhylovizLinearView(self.layoutOptions) + }, + + zoomAndPan : function(event){ + /** + * Function to zoom and pan the svg element which the entire tree is contained within + * Uses d3.zoom events, and extend them to allow manual updates and keeping states in model + */ + if (typeof event !== "undefined") { + var zoomParams = event.zoom, + translateParams = event.translate; + } + + var self = this, + scaleFactor = self.zoomFunc.scale(), + translationCoor = self.zoomFunc.translate(), + zoomStatement = "", + translateStatement = ""; + + // Do manual scaling. + switch (zoomParams) { + case "reset": + scaleFactor = 1.0; + translationCoor = [0,0]; break; + case "+": + scaleFactor *= 1.1; break; + case "-": + scaleFactor *= 0.9; break; + default: + if (typeof zoomParams === "number") { + scaleFactor = zoomParams; + } else if (d3.event !== null) { + scaleFactor = d3.event.scale; + } + } + if (scaleFactor < self.MIN_SCALE || scaleFactor > self.MAX_SCALE) { return;} + self.zoomFunc.scale(scaleFactor); //update scale Factor + zoomStatement = "translate(" + self.margins[3] + "," + self.margins[0] + ")" + + " scale(" + scaleFactor + ")"; + + // Do manual translation. + if( d3.event !== null) { + translateStatement = "translate(" + d3.event.translate + ")"; + } else { + if(typeof translateParams !== "undefined") { + var x = translateParams.split(",")[0]; + var y = translateParams.split(",")[1]; + if (!isNaN(x) && !isNaN(y)){ + translationCoor = [translationCoor[0] + parseFloat(x), translationCoor[1] + parseFloat(y)]; + } + } + self.zoomFunc.translate(translationCoor); // update zoomFunc + translateStatement = "translate(" + translationCoor + ")"; + } + + self.phyloTree.set("scaleFactor", scaleFactor); + self.phyloTree.set("translate", translationCoor); + self.vis.attr("transform", translateStatement + zoomStatement); //refers to the view that we are actually zooming + }, + + + reloadViz : function() { + /** + * Primes the Ajax URL to load another Nexus tree + */ + var self = this, + treeIndex = $("#phylovizNexSelector :selected").val(), + dataset_id = self.phyloTree.get("dataset_id"), + url = "phyloviz/getJsonData?dataset_id=" + dataset_id + "&treeIndex=" + String(treeIndex); + $.getJSON(url, function(packedJson){ + window.initPhyloViz(packedJson.data, packedJson.config); + }); + } +}); + + +var HeaderButtons = Backbone.View.extend({ + + initialize : function(phylovizView){ + var self = this; + self.phylovizView = phylovizView; + + // Clean up code - if the class initialized more than once + $("#panelHeaderRightBtns").empty(); + $("#phyloVizNavBtns").empty(); + $("#phylovizNexSelector").off(); + + self.initNavBtns(); + self.initRightHeaderBtns(); + + // Initial a tree selector in the case of nexus + $("#phylovizNexSelector").off().on("change", function() {self.phylovizView.reloadViz();} ); + + }, + + initRightHeaderBtns : function(){ + var self = this; + + rightMenu = create_icon_buttons_menu([ + { icon_class: 'gear', title: 'PhyloViz Settings', on_click: function(){ + $("#SettingsMenu").show(); + self.settingsMenu.updateUI(); + } }, + { icon_class: 'disk', title: 'Save visualization', on_click: function() { + var nexSelected = $("#phylovizNexSelector option:selected").text(); + if(nexSelected) { + self.phylovizView.phyloTree.set("title", nexSelected); + } + self.phylovizView.phyloTree.save(); + } }, + { icon_class: 'chevron-expand', title: 'Search / Edit Nodes', on_click: function() { + $("#nodeSelectionView").show(); + } }, + { icon_class: 'information', title: 'Phyloviz Help', on_click: function() { + window.open('http://wiki.g2.bx.psu.edu/Learn/Visualization/PhylogeneticTree'); + // https://docs.google.com/document/d/1AXFoJgEpxr21H3LICRs3EyMe1B1X_KFPouzIgrCz3zk/edit + } } + ], + { + tooltip_config: { placement: 'bottom' } + }); + $("#panelHeaderRightBtns").append(rightMenu.$el); + }, + + initNavBtns: function() { + var self = this, + navMenu = create_icon_buttons_menu([ + { icon_class: 'zoom-in', title: 'Zoom in', on_click: function() { + self.phylovizView.zoomAndPan({ zoom : "+"}); + } }, + { icon_class: 'zoom-out', title: 'Zoom out', on_click: function() { + self.phylovizView.zoomAndPan({ zoom : "-"}); + } }, + { icon_class: 'arrow-circle', title: 'Reset Zoom/Pan', on_click: function() { + self.phylovizView.zoomAndPan({ zoom : "reset"}); + } } + ], + { + tooltip_config: { placement: 'bottom' } + }); + $("#phyloVizNavBtns").append(navMenu.$el); + } +}); + + +var SettingsMenu = UserMenuBase.extend({ + + className: 'Settings', + + initialize: function(options){ + // settings needs to directly interact with the phyloviz model so it will get access to it. + var self = this; + self.phyloTree = options.phyloTree; + self.el = $("#SettingsMenu"); + self.inputs = { + separation : $("#phyloVizTreeSeparation"), + leafHeight : $("#phyloVizTreeLeafHeight"), + fontSize : $("#phyloVizTreeFontSize") + }; + + //init all buttons of settings + $("#settingsCloseBtn").off().on("click", function() { self.el.hide(); }); + $("#phylovizResetSettingsBtn").off().on("click", function() { self.resetToDefaults(); }); + $("#phylovizApplySettingsBtn").off().on("click", function() { self.apply(); }); + }, + + apply : function(){ + /** + * Applying user values to phylotree model. + */ + var self = this; + if (!self.isAcceptableValue(self.inputs["separation"], 50, 2500) || + !self.isAcceptableValue(self.inputs["leafHeight"], 5, 30) || + !self.isAcceptableValue(self.inputs["fontSize"], 5, 20)){ + return; + } + $.each(self.inputs, function(key, $input){ + self.phyloTree.set(key, $input.val()); + }); + }, + updateUI : function(){ + /** + * Called to update the values input to that stored in the model + */ + var self = this; + $.each(self.inputs, function(key, $input){ + $input.val(self.phyloTree.get(key)); + }); + }, + resetToDefaults : function(){ + /** + * Resets the value of the phyloTree model to its default + */ + $(".bs-tooltip").remove(); // just in case the tool tip was not removed + var self = this; + $.each(self.phyloTree.defaults, function(key, value) { + self.phyloTree.set(key, value); + }); + self.updateUI(); + }, + + render: function(){ + + } + +}); + + +var NodeSelectionView = UserMenuBase.extend({ + /** + * View for inspecting node properties and editing them + */ + className: 'Settings', + + initialize : function (options){ + var self = this; + self.el = $("#nodeSelectionView"); + self.phyloTree = options.phyloTree; + + self.UI = { + enableEdit : $('#phylovizEditNodesCheck'), + saveChanges : $('#phylovizNodeSaveChanges'), + cancelChanges : $("#phylovizNodeCancelChanges"), + name : $("#phyloVizSelectedNodeName"), + dist : $("#phyloVizSelectedNodeDist"), + annotation : $("#phyloVizSelectedNodeAnnotation") + }; + + self.valuesOfConcern = { + name : null, + dist : null, + annotation : null + }; // temporarily stores the values in case user change their mind + + //init UI buttons + $("#nodeSelCloseBtn").off().on("click", function() { self.el.hide(); }); + self.UI.saveChanges.off().on("click", function(){ self.updateNodes(); }); + self.UI.cancelChanges.off().on("click", function(){ self.cancelChanges(); }); + + (function ($) { + // extending jquery fxn for enabling and disabling nodes. + $.fn.enable = function (isEnabled) { + return $(this).each(function () { + if(isEnabled){ + $(this).removeAttr('disabled'); + } else { + $(this).attr('disabled', 'disabled'); + } + }); + }; + })(jQuery); + + self.UI.enableEdit.off().on("click", function () { + self.toggleUI(); + }); + }, + + toggleUI : function(){ + /** + * For turning on and off the child elements + */ + var self = this, + checked = self.UI.enableEdit.is(':checked'); + + !checked ? self.cancelChanges() : ""; + + $.each(self.valuesOfConcern, function(key, value) { + self.UI[key].enable(checked); + }); + if(checked){ + self.UI.saveChanges.show(); + self.UI.cancelChanges.show(); + } else { + self.UI.saveChanges.hide(); + self.UI.cancelChanges.hide(); + } + + }, + + cancelChanges : function() { + /** + * Reverting to previous values in case user change their minds + */ + var self = this, + node = self.phyloTree.get("selectedNode"); + if (node){ + $.each(self.valuesOfConcern, function(key, value) { + self.UI[key].val(node[key]); + }); + } + }, + + updateNodes : function (){ + /** + * Changing the data in the underlying tree with user-specified values + */ + var self = this, + node = self.phyloTree.get("selectedNode"); + if (node){ + if (!self.isAcceptableValue(self.UI.dist, 0, 1) || + self.hasIllegalJsonCharacters(self.UI.name) || + self.hasIllegalJsonCharacters(self.UI.annotation) ) { + return; + } + $.each(self.valuesOfConcern, function(key, value) { + (node[key]) = self.UI[key].val(); + }); + self.phyloTree.set("nodeAttrChangedTime", new Date()); + } else { + alert("No node selected"); + } + } + + +}); + + + +var PhyloVizSearch = UserMenuBase.extend({ + /** + * Initializes the search panel on phyloviz and handles its user interaction + * It allows user to search the entire free based on some qualifer, like dist <= val. + */ + initialize : function () { + var self = this; + + $("#phyloVizSearchBtn").on("click", function(){ + var searchTerm = $("#phyloVizSearchTerm"), + searchConditionVal = $("#phyloVizSearchCondition").val().split("-"), + attr = searchConditionVal[0], + condition = searchConditionVal[1]; + self.hasIllegalJsonCharacters(searchTerm); + + if (attr === "dist"){ + self.isAcceptableValue(searchTerm, 0, 1); + } + self.searchTree(attr, condition, searchTerm.val()); + }); + }, + + searchTree : function (attr, condition, val){ + /** + * Searches the entire tree and will highlight the nodes that match the condition in green + */ + d3.selectAll("g.node") + .classed("searchHighlight", function(d){ + var attrVal = d[attr]; + if (typeof attrVal !== "undefined" && attrVal !== null){ + if (attr === "dist"){ + switch (condition) { + case "greaterEqual": + return attrVal >= +val; + case "lesserEqual": + return attrVal <= +val; + default: + return; + } + + } else if (attr === "name" || attr === "annotation") { + return attrVal.toLowerCase().indexOf(val.toLowerCase()) !== -1; + } + } + }); + } +}); \ No newline at end of file diff --git a/templates/root/history.mako b/templates/root/history.mako index 10e7e34b1b7..edddf1015e0 100644 --- a/templates/root/history.mako +++ b/templates/root/history.mako @@ -272,6 +272,17 @@ $(function() { } init_trackster_links(); + + function init_phyloviz_links() { + // PhyloViz links + // Add to trackster browser functionality + $(".phyloviz-add").live("click", function() { + var dataset = this, + dataset_jquery = $(this); + window.parent.location = dataset_jquery.attr("new-url"); + }); + } + init_phyloviz_links(); // History rename functionality. async_save_text("history-name-container", "history-name", "${h.url_for( controller="/history", action="rename_async", id=trans.security.encode_id(history.id) )}", "new_name", 18); diff --git a/templates/root/history_common.mako b/templates/root/history_common.mako index be2c82b8ed8..114144e2eac 100644 --- a/templates/root/history_common.mako +++ b/templates/root/history_common.mako @@ -29,6 +29,9 @@ ## Render the dataset `data` as history item, using `hid` as the displayed id <%def name="render_dataset( data, hid, show_deleted_on_refresh = False, for_editing = True, display_structured = False )"> <% + + from galaxy.datatypes.xml import Phyloxml + from galaxy.datatypes.data import Newick, Nexus dataset_id = trans.security.encode_id( data.id ) if data.state in ['no state','',None]: @@ -230,6 +233,14 @@ action-url="${h.url_for( controller='tracks', action='browser', dataset_id=dataset_id)}" new-url="${h.url_for( controller='tracks', action='index', dataset_id=dataset_id, default_dbkey=data.dbkey)}" title="View in Trackster"> %endif + <% + isPhylogenyData = isinstance(data.datatype, (Phyloxml, Nexus, Newick)) + %> + %if isPhylogenyData: + + %endif %if trans.user: %if not display_structured:
diff --git a/templates/visualization/phyloviz.mako b/templates/visualization/phyloviz.mako new file mode 100644 index 00000000000..bcbb3e4608e --- /dev/null +++ b/templates/visualization/phyloviz.mako @@ -0,0 +1,320 @@ +<%inherit file="/webapps/galaxy/base_panels.mako"/> +## +<%def name="init()"> + <% + self.has_left_panel=False + self.has_right_panel=False + self.active_view="visualization" + self.message_box_visible=False + %> + + +<%def name="stylesheets()"> + ${parent.stylesheets()} + + + + +<%def name="javascripts()"> + ${parent.javascripts()} + ${h.js( "galaxy.panels", "libs/d3", "mvc/data", "viz/visualization", "viz/phyloviz")} + + + + +<%def name="center_panel()"> + +
+
+
+
+
+
+
+ + +
+
+ %if config["ext"] == "nex" and not config["saved_visualization"]: + + %endif + +
+

 | Alt+click to select nodes

+
+ + +
+ +
+ + ## Node Selection Menu +
+
+

Search / Edit Nodes : + X +

+
+ +
+ +
+ Search for nodes with: + + + +
+ +
+
+ +
+ +
+ Name: +
+
+ Dist: +
+
+ Annotation: + +
+
+ Edit: + + +
+
+
+ + ## Settings Menus +
+
+

Phyloviz Settings: + X +

+
+
+
+ Phylogenetic Spacing (px per unit): (50-2500) +
+
+ Vertical Spacing (px): (5-30) +
+
+ Font Size (px): (5-20) +
+ +
+
+ + +
+
+ + + + + + + + +
+
+ + + + + + From cb0614f0fa7e480907c33b784283b2116a3c3173 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Mon, 27 Aug 2012 12:52:43 -0400 Subject: [PATCH 02/16] Fixed truncation of downloaded fasta files. --- lib/galaxy/jobs/deferred/genome_transfer.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/jobs/deferred/genome_transfer.py b/lib/galaxy/jobs/deferred/genome_transfer.py index 2fb9a4719cf..9ea720ecea9 100644 --- a/lib/galaxy/jobs/deferred/genome_transfer.py +++ b/lib/galaxy/jobs/deferred/genome_transfer.py @@ -115,15 +115,16 @@ class GenomeTransferPlugin( DataTransfer ): files = tar.getmembers() for filename in files: z = tar.extractfile(filename) - try: - chunk = z.read( CHUNK_SIZE ) - except IOError: - os.close( fd ) - log.error( 'Problem decompressing compressed data' ) - exit() - if not chunk: - break - os.write( fd, chunk ) + while 1: + try: + chunk = z.read( CHUNK_SIZE ) + except IOError: + os.close( fd ) + log.error( 'Problem decompressing compressed data' ) + exit() + if not chunk: + break + os.write( fd, chunk ) os.write( fd, '\n' ) os.close( fd ) tar.close() From ce86cf27155799598d3340c6e68aca39f3c6c77a Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Mon, 27 Aug 2012 15:44:01 -0400 Subject: [PATCH 03/16] Comment out phyloviz XML parser until native Python XML parsing is used. --- .../visualization/phyloviz/phyloviz_dataprovider.py | 8 ++++---- lib/galaxy/visualization/phyloviz/phyloxmlparser.py | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py b/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py index 026f33e51e6..017cd5e85a7 100644 --- a/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py +++ b/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py @@ -1,6 +1,6 @@ from newickparser import Newick_Parser from nexusparser import Nexus_Parser -from phyloxmlparser import Phyloxml_Parser +#from phyloxmlparser import Phyloxml_Parser class Phyloviz_DataProvider(object): @@ -16,9 +16,9 @@ class Phyloviz_DataProvider(object): if fileExt == "nhx": # parses newick files newickParser = Newick_Parser() jsonDicts, parseMsg = newickParser.parseFile(filepath) - elif fileExt == "phyloxml": # parses phyloXML files - phyloxmlParser = Phyloxml_Parser() - jsonDicts, parseMsg = phyloxmlParser.parseFile(filepath) + #elif fileExt == "phyloxml": # parses phyloXML files + # phyloxmlParser = Phyloxml_Parser() + # jsonDicts, parseMsg = phyloxmlParser.parseFile(filepath) elif fileExt == "nex": # parses nexus files nexusParser = Nexus_Parser() jsonDicts, parseMsg = nexusParser.parseFile(filepath) diff --git a/lib/galaxy/visualization/phyloviz/phyloxmlparser.py b/lib/galaxy/visualization/phyloviz/phyloxmlparser.py index ddfea8c30e0..d8985103daa 100644 --- a/lib/galaxy/visualization/phyloviz/phyloxmlparser.py +++ b/lib/galaxy/visualization/phyloviz/phyloxmlparser.py @@ -1,4 +1,7 @@ from baseparser import Base_Parser, PhyloTree, Node + +''' +TOD0: use native Python XML parser. from lxml import etree class Phyloxml_Parser(Base_Parser): @@ -143,3 +146,4 @@ if __name__=="__main__": # filepath = "../data/" +"bcl_2.xml" # filepath = "../data/" +"reducedXml.xml" parser.parseFile(filepath) +''' \ No newline at end of file From e9c26d05d30b7b3518e2db1b57a3385c9451a646 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Tue, 28 Aug 2012 09:05:31 -0400 Subject: [PATCH 04/16] Fixes for phyloviz parser and a4c7aeb61c7b. --- .../phyloviz/phyloviz_dataprovider.py | 8 +++---- .../visualization/phyloviz/phyloxmlparser.py | 21 +++---------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py b/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py index 017cd5e85a7..026f33e51e6 100644 --- a/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py +++ b/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py @@ -1,6 +1,6 @@ from newickparser import Newick_Parser from nexusparser import Nexus_Parser -#from phyloxmlparser import Phyloxml_Parser +from phyloxmlparser import Phyloxml_Parser class Phyloviz_DataProvider(object): @@ -16,9 +16,9 @@ class Phyloviz_DataProvider(object): if fileExt == "nhx": # parses newick files newickParser = Newick_Parser() jsonDicts, parseMsg = newickParser.parseFile(filepath) - #elif fileExt == "phyloxml": # parses phyloXML files - # phyloxmlParser = Phyloxml_Parser() - # jsonDicts, parseMsg = phyloxmlParser.parseFile(filepath) + elif fileExt == "phyloxml": # parses phyloXML files + phyloxmlParser = Phyloxml_Parser() + jsonDicts, parseMsg = phyloxmlParser.parseFile(filepath) elif fileExt == "nex": # parses nexus files nexusParser = Nexus_Parser() jsonDicts, parseMsg = nexusParser.parseFile(filepath) diff --git a/lib/galaxy/visualization/phyloviz/phyloxmlparser.py b/lib/galaxy/visualization/phyloviz/phyloxmlparser.py index d8985103daa..ab8d00cd07b 100644 --- a/lib/galaxy/visualization/phyloviz/phyloxmlparser.py +++ b/lib/galaxy/visualization/phyloviz/phyloxmlparser.py @@ -1,8 +1,5 @@ from baseparser import Base_Parser, PhyloTree, Node - -''' -TOD0: use native Python XML parser. -from lxml import etree +from xml.etree import ElementTree class Phyloxml_Parser(Base_Parser): """Parses a phyloxml file into a json file that will be passed to PhyloViz for display""" @@ -22,7 +19,7 @@ class Phyloxml_Parser(Base_Parser): """passes a file and extracts its Phylogeny Tree content.""" phyloXmlFile = open(filePath, "r") - xmlTree = etree.parse(phyloXmlFile) + xmlTree = ElementTree.parse(phyloXmlFile) xmlRoot = xmlTree.getroot()[0] self.nameSpaceIndex = xmlRoot.tag.rfind("}") + 1 # used later by the clean tag method to remove the name space in every element.tag @@ -134,16 +131,4 @@ class Phyloxml_Parser(Base_Parser): def cleanTag(self, tagString): return tagString[self.nameSpaceIndex:] - - -if __name__=="__main__": - - # Files tested against - parser = Phyloxml_Parser() - filepath = "../data/" +"apaf.xml" - # filepath = "../data/" +"12_multiple_supports.xml" - - # filepath = "../data/" +"bcl_2.xml" - # filepath = "../data/" +"reducedXml.xml" - parser.parseFile(filepath) -''' \ No newline at end of file + \ No newline at end of file From f047aa99e86cf160cdad0faae028b367de1b982e Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Tue, 28 Aug 2012 11:01:07 -0400 Subject: [PATCH 05/16] Add the ability to browse writable repositories in a tool shed. --- .../community/controllers/repository.py | 25 +++++++- templates/webapps/community/index.mako | 57 +++++++++++-------- 2 files changed, 56 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/webapps/community/controllers/repository.py b/lib/galaxy/webapps/community/controllers/repository.py index 5526d5427a3..f99025fd700 100644 --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -246,6 +246,25 @@ class EmailAlertsRepositoryListGrid( RepositoryListGrid ): grids.GridAction( "User preferences", dict( controller='user', action='index', cntrller='repository', webapp='community' ) ) ] +class WritableRepositoryListGrid( RepositoryListGrid ): + def build_initial_query( self, trans, **kwd ): + # TODO: improve performance by adding a db table associating users with repositories for which they have write access. + username = kwd[ 'username' ] + clause_list = [] + for repository in trans.sa_session.query( self.model_class ): + allow_push_usernames = repository.allow_push.split( ',' ) + if username in allow_push_usernames: + clause_list.append( self.model_class.table.c.id == repository.id ) + if clause_list: + return trans.sa_session.query( self.model_class ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + # Return an empty query. + return trans.sa_session.query( self.model_class ) \ + .filter( self.model_class.table.c.id < 0 ) + class ValidRepositoryListGrid( RepositoryListGrid ): class CategoryColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): @@ -393,6 +412,7 @@ class RepositoryController( BaseUIController, ItemRatings ): email_alerts_repository_list_grid = EmailAlertsRepositoryListGrid() category_list_grid = CategoryListGrid() valid_category_list_grid = ValidCategoryListGrid() + writable_repository_list_grid = WritableRepositoryListGrid() def __add_hgweb_config_entry( self, trans, repository, repository_path ): # Add an entry in the hgweb.config file for a new repository. An entry looks something like: @@ -519,12 +539,15 @@ class RepositoryController( BaseUIController, ItemRatings ): repository_id = kwd.get( 'id', None ) repository = get_repository( trans, repository_id ) kwd[ 'f-email' ] = repository.user.email - elif operation == "my_repositories": + elif operation == "repositories_i_own": # Eliminate the current filters if any exist. for k, v in kwd.items(): if k.startswith( 'f-' ): del kwd[ k ] kwd[ 'f-email' ] = trans.user.email + elif operation == "writable_repositories": + kwd[ 'username' ] = trans.user.username + return self.writable_repository_list_grid( trans, **kwd ) elif operation == "repositories_by_category": # Eliminate the current filters if any exist. for k, v in kwd.items(): diff --git a/templates/webapps/community/index.mako b/templates/webapps/community/index.mako index 6e8c293e660..251f5ff71a2 100644 --- a/templates/webapps/community/index.mako +++ b/templates/webapps/community/index.mako @@ -60,34 +60,41 @@ %endif
- Repositories + All Repositories
-
-
-
- Browse by category -
- %if trans.user: -
- Browse my repositories -
-
- Browse my invalid tools -
- %endif +
+ Browse by category +
+ %if trans.user: +
+
+ My Repositories and Tools
-
-
-
-
- %if trans.user: - Create new repository - %else: - Login to create a repository - %endif -
+
+ Repositories I own
-
+
+ My writable repositories +
+
+ My invalid tools +
+
+
+ Available Actions +
+
+ Create new repository +
+ %else: +
+
+ Available Actions +
+
+ Login to create a repository +
+ %endif
From 112d13e066cac420e5d1356c82796f1144ce4a88 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 28 Aug 2012 11:32:17 -0400 Subject: [PATCH 06/16] Improved generation of tool shed URLs --- lib/galaxy/util/shed_util.py | 13 +++++--- lib/galaxy/web/controllers/admin_toolshed.py | 32 ++++++++++---------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/util/shed_util.py b/lib/galaxy/util/shed_util.py index cf70499c2e7..591ed16aaae 100644 --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -454,7 +454,7 @@ def create_tool_dependency_objects( app, tool_shed_repository, relative_install_ def generate_clone_url( trans, repository ): """Generate the URL for cloning a repository.""" tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) + return url_join( tool_shed_url, 'repos', repository.owner, repository.name ) def generate_datatypes_metadata( datatypes_config, metadata_dict ): """Update the received metadata_dict with information from the parsed datatypes_config.""" tree = ElementTree.parse( datatypes_config ) @@ -993,7 +993,7 @@ def get_converter_and_display_paths( registration_elem, relative_install_dir ): break return converter_path, display_path def get_ctx_rev( tool_shed_url, name, owner, changeset_revision ): - url = '%s/repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % ( tool_shed_url, name, owner, changeset_revision ) + url = url_join( tool_shed_url, 'repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % ( name, owner, changeset_revision ) ) response = urllib2.urlopen( url ) ctx_rev = response.read() response.close() @@ -1221,8 +1221,8 @@ def get_tool_version_association( app, parent_tool_version, tool_version ): def get_update_to_changeset_revision_and_ctx_rev( trans, repository ): """Return the changeset revision hash to which the repository can be updated.""" tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = '%s/repository/get_changeset_revision_and_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % \ - ( tool_shed_url, repository.name, repository.owner, repository.installed_changeset_revision ) + url = url_join( tool_shed_url, 'repository/get_changeset_revision_and_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % \ + ( repository.name, repository.owner, repository.installed_changeset_revision ) ) try: response = urllib2.urlopen( url ) encoded_update_dict = response.read() @@ -1645,3 +1645,8 @@ def update_tool_shed_repository_status( app, tool_shed_repository, status ): tool_shed_repository.status = status sa_session.add( tool_shed_repository ) sa_session.flush() +def url_join( *args ): + parts = [] + for arg in args: + parts.append( arg.strip( '/' ) ) + return '/'.join( parts ) diff --git a/lib/galaxy/web/controllers/admin_toolshed.py b/lib/galaxy/web/controllers/admin_toolshed.py index ca40903d355..e524fa1615d 100644 --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -374,7 +374,7 @@ class AdminToolshed( AdminGalaxy ): def browse_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] galaxy_url = url_for( '/', qualified=True ) - url = '%srepository/browse_valid_categories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + url = url_join( tool_shed_url, 'repository/browse_valid_categories?galaxy_url=%s&webapp=galaxy' % ( galaxy_url ) ) return trans.response.send_redirect( url ) @web.expose @web.require_admin @@ -392,8 +392,8 @@ class AdminToolshed( AdminGalaxy ): # Send a request to the relevant tool shed to see if there are any updates. repository = get_repository( trans, kwd[ 'id' ] ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) + url = url_join( tool_shed_url, 'repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) ) return trans.response.send_redirect( url ) @web.expose @web.require_admin @@ -467,14 +467,14 @@ class AdminToolshed( AdminGalaxy ): def find_tools_in_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] galaxy_url = url_for( '/', qualified=True ) - url = '%srepository/find_tools?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + url = url_join( tool_shed_url, 'repository/find_tools?galaxy_url=%s&webapp=galaxy' % galaxy_url ) return trans.response.send_redirect( url ) @web.expose @web.require_admin def find_workflows_in_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] galaxy_url = url_for( '/', qualified=True ) - url = '%srepository/find_workflows?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + url = url_join( tool_shed_url, 'repository/find_workflows?galaxy_url=%s&webapp=galaxy' % galaxy_url ) return trans.response.send_redirect( url ) def generate_tool_path( self, repository_clone_url, changeset_revision ): """ @@ -489,7 +489,7 @@ class AdminToolshed( AdminGalaxy ): tool_shed_url = items[ 0 ] repo_path = items[ 1 ] tool_shed_url = clean_tool_shed_url( tool_shed_url ) - return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) + return url_join( tool_shed_url, 'repos', repo_path, changeset_revision ) @web.json @web.require_admin def get_file_contents( self, trans, file_path ): @@ -634,8 +634,8 @@ class AdminToolshed( AdminGalaxy ): tool_shed_repository, trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) + url = url_join( tool_shed_url, '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) ) response = urllib2.urlopen( url ) text = response.read() response.close() @@ -954,7 +954,7 @@ class AdminToolshed( AdminGalaxy ): repository_ids = kwd.get( 'repository_ids', None ) changeset_revisions = kwd.get( 'changeset_revisions', None ) # Get the information necessary to install each repository. - url = '%srepository/get_repository_information?repository_ids=%s&changeset_revisions=%s&webapp=galaxy' % ( tool_shed_url, repository_ids, changeset_revisions ) + url = url_join( tool_shed_url, 'repository/get_repository_information?repository_ids=%s&changeset_revisions=%s&webapp=galaxy' % ( repository_ids, changeset_revisions ) ) response = urllib2.urlopen( url ) raw_text = response.read() response.close() @@ -1097,8 +1097,8 @@ class AdminToolshed( AdminGalaxy ): name = repo_info_dict.keys()[ 0 ] repo_info_tuple = repo_info_dict[ name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple - url = '%srepository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, name, repository_owner, changeset_revision ) + url = url_join( tool_shed_url, 'repository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( name, repository_owner, changeset_revision ) ) response = urllib2.urlopen( url ) raw_text = response.read() response.close() @@ -1273,8 +1273,8 @@ class AdminToolshed( AdminGalaxy ): tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) # Get all previous change set revisions from the tool shed for the repository back to, but excluding, the previous valid changeset # revision to see if it was previously installed using one of them. - url = '%s/repository/previous_changeset_revisions?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, url_for( '/', qualified=True ), repository_name, repository_owner, changeset_revision ) + url = url_join( tool_shed_url, 'repository/previous_changeset_revisions?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( url_for( '/', qualified=True ), repository_name, repository_owner, changeset_revision ) ) response = urllib2.urlopen( url ) text = response.read() response.close() @@ -1350,8 +1350,8 @@ class AdminToolshed( AdminGalaxy ): # Get the tool_versions from the tool shed for each tool in the installed change set. repository = get_repository( trans, kwd[ 'id' ] ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, repository.name, repository.owner, repository.changeset_revision ) + url = url_join( tool_shed_url, 'repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( repository.name, repository.owner, repository.changeset_revision ) ) response = urllib2.urlopen( url ) text = response.read() response.close() @@ -1522,7 +1522,7 @@ class AdminToolshed( AdminGalaxy ): def __generate_clone_url( self, trans, repository ): """Generate the URL for cloning a repository.""" tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) + return url_join( tool_shed_url, 'repos', repository.owner, repository.name ) ## ---- Utility methods ------------------------------------------------------- From c68066fac2efa33bf3f8afa4fb8674dd7f5aff21 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Tue, 28 Aug 2012 11:49:43 -0400 Subject: [PATCH 07/16] Apply the improvements for generation of tool shed URLs to the repository controller. --- lib/galaxy/web/controllers/admin_toolshed.py | 29 ++++++++++++------- .../community/controllers/repository.py | 16 +++++----- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/web/controllers/admin_toolshed.py b/lib/galaxy/web/controllers/admin_toolshed.py index e524fa1615d..863f36ca508 100644 --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -392,8 +392,9 @@ class AdminToolshed( AdminGalaxy ): # Send a request to the relevant tool shed to see if there are any updates. repository = get_repository( trans, kwd[ 'id' ] ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = url_join( tool_shed_url, 'repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) ) + url = url_join( tool_shed_url, + 'repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) ) return trans.response.send_redirect( url ) @web.expose @web.require_admin @@ -634,8 +635,9 @@ class AdminToolshed( AdminGalaxy ): tool_shed_repository, trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) - url = url_join( tool_shed_url, '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) ) + url = url_join( tool_shed_url, + '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) ) response = urllib2.urlopen( url ) text = response.read() response.close() @@ -954,7 +956,9 @@ class AdminToolshed( AdminGalaxy ): repository_ids = kwd.get( 'repository_ids', None ) changeset_revisions = kwd.get( 'changeset_revisions', None ) # Get the information necessary to install each repository. - url = url_join( tool_shed_url, 'repository/get_repository_information?repository_ids=%s&changeset_revisions=%s&webapp=galaxy' % ( repository_ids, changeset_revisions ) ) + url = url_join( tool_shed_url, + 'repository/get_repository_information?repository_ids=%s&changeset_revisions=%s&webapp=galaxy' % \ + ( repository_ids, changeset_revisions ) ) response = urllib2.urlopen( url ) raw_text = response.read() response.close() @@ -1097,8 +1101,9 @@ class AdminToolshed( AdminGalaxy ): name = repo_info_dict.keys()[ 0 ] repo_info_tuple = repo_info_dict[ name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple - url = url_join( tool_shed_url, 'repository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( name, repository_owner, changeset_revision ) ) + url = url_join( tool_shed_url, + 'repository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( name, repository_owner, changeset_revision ) ) response = urllib2.urlopen( url ) raw_text = response.read() response.close() @@ -1273,8 +1278,9 @@ class AdminToolshed( AdminGalaxy ): tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) # Get all previous change set revisions from the tool shed for the repository back to, but excluding, the previous valid changeset # revision to see if it was previously installed using one of them. - url = url_join( tool_shed_url, 'repository/previous_changeset_revisions?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( url_for( '/', qualified=True ), repository_name, repository_owner, changeset_revision ) ) + url = url_join( tool_shed_url, + 'repository/previous_changeset_revisions?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( url_for( '/', qualified=True ), repository_name, repository_owner, changeset_revision ) ) response = urllib2.urlopen( url ) text = response.read() response.close() @@ -1350,8 +1356,9 @@ class AdminToolshed( AdminGalaxy ): # Get the tool_versions from the tool shed for each tool in the installed change set. repository = get_repository( trans, kwd[ 'id' ] ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = url_join( tool_shed_url, 'repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( repository.name, repository.owner, repository.changeset_revision ) ) + url = url_join( tool_shed_url, + 'repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( repository.name, repository.owner, repository.changeset_revision ) ) response = urllib2.urlopen( url ) text = response.read() response.close() diff --git a/lib/galaxy/webapps/community/controllers/repository.py b/lib/galaxy/webapps/community/controllers/repository.py index f99025fd700..01468d9124e 100644 --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -11,7 +11,7 @@ from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * from galaxy.util.shed_util import create_repo_info_dict, get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, NOT_TOOL_CONFIGS from galaxy.util.shed_util import open_repository_files_folder, reversed_lower_upper_bounded_changelog, reversed_upper_bounded_changelog, strip_path -from galaxy.util.shed_util import to_html_escaped, update_repository +from galaxy.util.shed_util import to_html_escaped, update_repository, url_join from galaxy.tool_shed.encoding_util import * from common import * @@ -749,9 +749,10 @@ class RepositoryController( BaseUIController, ItemRatings ): update = 'true' no_update = 'false' else: - # Start building up the url to redirect back to the calling Galaxy instance. - url = '%sadmin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '/', qualified=True ) ) - url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % ( repository.name, repository.user.username, changeset_revision ) + # Start building up the url to redirect back to the calling Galaxy instance. + url = url_join( galaxy_url, + 'admin_toolshed/update_to_changeset_revision?tool_shed_url=%s&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ + ( url_for( '/', qualified=True ), repository.name, repository.user.username, changeset_revision ) ) if changeset_revision == repository.tip: # If changeset_revision is the repository tip, there are no additional updates. if from_update_manager: @@ -1395,10 +1396,9 @@ class RepositoryController( BaseUIController, ItemRatings ): """Send the list of repository_ids and changeset_revisions to Galaxy so it can begin the installation process.""" galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) # Redirect back to local Galaxy to perform install. - url = '%sadmin_toolshed/prepare_for_install' % galaxy_url - url += '?tool_shed_url=%s' % url_for( '/', qualified=True ) - url += '&repository_ids=%s' % ','.join( util.listify( repository_ids ) ) - url += '&changeset_revisions=%s' % ','.join( util.listify( changeset_revisions ) ) + url = url_join( galaxy_url, + 'admin_toolshed/prepare_for_install?tool_shed_url=%s&repository_ids=%s&changeset_revisions=%s' % \ + ( url_for( '/', qualified=True ), ','.join( util.listify( repository_ids ) ), ','.join( util.listify( changeset_revisions ) ) ) ) return trans.response.send_redirect( url ) @web.expose def load_invalid_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ): From acb8767200f9e9bd5933e053e1baae8c37c98d89 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 28 Aug 2012 13:50:54 -0400 Subject: [PATCH 08/16] Don't attempt to run util.umask_fix_perms() on data that is being linked to. Fixes #801. --- lib/galaxy/jobs/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 50c185834c2..99cdb7e5088 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -471,7 +471,7 @@ class JobWrapper( object ): job.user.total_disk_usage += bytes # fix permissions - for path in [ dp.real_path for dp in self.get_output_fnames() ]: + for path in [ dp.real_path for dp in self.get_mutable_output_fnames() ]: util.umask_fix_perms( path, self.app.config.umask, 0666, self.app.config.gid ) self.sa_session.flush() log.debug( 'job %d ended' % self.job_id ) @@ -679,6 +679,11 @@ class JobWrapper( object ): self.compute_outputs() return self.output_paths + def get_mutable_output_fnames( self ): + if self.output_paths is None: + self.compute_outputs() + return filter( lambda dsp: dsp.mutable, self.output_paths ) + def get_output_hdas_and_fnames( self ): if self.output_hdas_and_paths is None: self.compute_outputs() @@ -686,10 +691,11 @@ class JobWrapper( object ): def compute_outputs( self ) : class DatasetPath( object ): - def __init__( self, dataset_id, real_path, false_path = None ): + def __init__( self, dataset_id, real_path, false_path = None, mutable = True ): self.dataset_id = dataset_id self.real_path = real_path self.false_path = false_path + self.mutable = mutable def __str__( self ): if self.false_path is None: return self.real_path @@ -706,13 +712,13 @@ class JobWrapper( object ): self.output_hdas_and_paths = {} for name, hda in [ ( da.name, da.dataset ) for da in job.output_datasets + job.output_library_datasets ]: false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % hda.dataset.id ) ) - dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path ) + dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path, mutable = hda.dataset.external_filename is None ) self.output_paths.append( dsp ) self.output_hdas_and_paths[name] = hda, dsp if special: false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % special.dataset.id ) ) else: - results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name ) ) for da in job.output_datasets + job.output_library_datasets ] + results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name, mutable = da.dataset.dataset.external_filename is None ) ) for da in job.output_datasets + job.output_library_datasets ] self.output_paths = [t[2] for t in results] self.output_hdas_and_paths = dict([(t[0], t[1:]) for t in results]) if special: From 131fb1e226e586bcf5d1b5c425a2cbe813c1290f Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Tue, 28 Aug 2012 14:32:04 -0400 Subject: [PATCH 09/16] Fix handling of ToolSectionLabel objects in the tool panel. --- lib/galaxy/tools/__init__.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index f686a9216a9..e3e64a69b3c 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -187,7 +187,9 @@ class ToolBox( object ): section.elems[ section_key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) elif section_key.startswith( 'label_' ): - section.elems[ section_key ] = section_val + if section_val: + section.elems[ section_key ] = section_val + log.debug( "Loaded label: %s" % ( section_val.text ) ) self.tool_panel[ key ] = section def load_integrated_tool_panel_keys( self ): """ @@ -215,12 +217,12 @@ class ToolBox( object ): section.elems[ key ] = None elif section_elem.tag == 'label': key = 'label_%s' % section_elem.get( 'id' ) - section.elems[ key ] = ToolSectionLabel( section_elem ) + section.elems[ key ] = None key = 'section_%s' % elem.get( 'id' ) self.integrated_tool_panel[ key ] = section elif elem.tag == 'label': key = 'label_%s' % elem.get( 'id' ) - self.integrated_tool_panel[ key ] = ToolSectionLabel( elem ) + self.integrated_tool_panel[ key ] = None def write_integrated_tool_panel_config_file( self ): """ Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators @@ -254,10 +256,11 @@ class ToolBox( object ): if section_item: os.write( fd, ' \n' % section_item.id ) elif section_key.startswith( 'label_' ): - label_id = section_item.id or '' - label_text = section_item.text or '' - label_version = section_item.version or '' - os.write( fd, '