opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into bbi-js3

Browse Source
This commit is contained in:
jgoecks
2016-08-19 17:38:25 -04:00
parent de763bedee bee9ebdbc1
commit cdc82ef2f8
54 changed files with 695 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/flake8_lint_include_list.txt
+1
View File
@@ -298,6 +298,7 @@ lib/galaxy/util/multi_byte.py
lib/galaxy/util/odict.py
lib/galaxy/util/pastescript/__init__.py
lib/galaxy/util/plugin_config.py
lib/galaxy/util/postfork.py
lib/galaxy/util/simplegraph.py
lib/galaxy_utils/__init__.py
lib/galaxy/util/sleeper.py
client/galaxy/style/less/library.less
+85 -146
View File
@@ -8,10 +8,6 @@ margin: auto;
margin-top:2em;
overflow: auto !important;
tr {
height: 32px;
}
.fa{
font-size: 12px;
}
@@ -19,224 +15,110 @@ tr {
font-size: initial;
margin-left: 0.6em;
}
.libraryRow {
.libraryRow{
background-color: @table-heading-bg;
}
.datasetHighlighted {
.datasetHighlighted{
background-color: @table-bg-accent;
}
.libraryItemDeleted-True {
.libraryItemDeleted-True{
font-style: italic;
}
div.libraryItemBody {
div.libraryItemBody{
padding: 4px 4px 2px 4px;
}
li.folderRow,
li.datasetRow
{
li.folderRow, li.datasetRow{
border-top: solid 1px @table-border;
}
li.folderRow:hover,
li.datasetRow:hover
{
li.folderRow:hover, li.datasetRow:hover{
background-color: @table-bg-accent;
}
td.right-center {
td.right-center{
vertical-align: middle !important;
text-align: right;
}
.library-genome-select {
.library-genome-select{
max-width: 350px;
}
.library-extension-select {
.library-extension-select{
max-width: 140px;
}
.library_table {
td {
.library_table{
td{
border-top:1px solid #5f6990 !important;
}
th {
th{
border-bottom: 2px solid #5f6990 !important;
}
a {
a{
color: #0A143D;
&:hover{
color: maroon;
}
}
}
tr.light td
{
tr.light td{
background-color: white;
color: black;
}
tr.light:hover td
{
tr.light:hover td{
background-color: #f5e8cc;
}
tr.dark td
{
tr.dark td{
background-color: #d6b161;
color: white;
}
tr.dark:hover td
{
tr.dark:hover td{
background-color: #ebd4a4;
color: white;
}
a.dark
{
a.dark{
color: white;
}
.dataset_table{
tr {
border-bottom: 1px solid #5f6990 !important;
}
th{
border: none !important;
}
td{
border: none !important;
tr, th, td{
border: none;
}
.dataset-first-column{
width: 30%;
}
}
th.button_heading{
width: 2em;
}
.bigdrop.select2-container .select2-results {
.bigdrop.select2-container .select2-results{
max-height: 300px;
}
.bigdrop .select2-results {
.bigdrop .select2-results{
max-height: 300px;
}
.select2-container-multi{
width: 100%;
}
.roles-selection {
.roles-selection{
width: 66%;
}
#library_toolbar {
#library_toolbar{
margin-bottom: 0.5em;
span {
span{
margin-right: 0.2em;
}
.toolbar-item{
margin-left: 1em;
}
}
#libraries_element, #folder_items_element{
button{
margin-left: 0.5em;
}
}
img.expanderIcon {
padding-right: 4px;
}
input.datasetCheckbox,
li, ul {
padding: 0;
margin: 0;
}
.rowTitle {
padding: 2px;
}
ul {
list-style: none;
}
.libraryTitle th {
text-align: left;
}
pre.peek {
background: white;
color: black;
// width: 100%;
overflow: auto;
}
pre.peek th {
color: white;
background: @table-heading-bg;
}
.help-button {
.help-button{
float: right;
}
span.expandLink {
padding-left: 12px;
display: inline-block;
vertical-align: middle;
background: url(../images/silk/resultset_next.png) no-repeat;
}
.folderRow.expanded span.expandLink {
background: url(../images/silk/resultset_bottom.png) no-repeat;
}
.folderRow span.rowIcon {
float: left;
margin-right: 5px;
width: 16px;
height: 16px;
display: inline-block;
vertical-align: middle;
background: url(../images/silk/folder.png);
}
.libraryItem-error {
margin-right: 2px;
padding: 0 2px 0 2px;
border: 1px solid @state-danger-border;
background: @state-danger-bg;
}
.libraryItem-queued {
margin-right: 2px;
padding: 0 2px 0 2px;
border: 1px solid @state-default-border;
background: @state-default-bg;
}
.libraryItem-running {
margin-right: 2px;
padding: 0 2px 0 2px;
border: 1px solid @state-running-border;
background: @state-running-bg;
}
.libraryItem-upload {
margin-right: 2px;
padding: 0 2px 0 2px;
border: 1px solid @state-upload-border;
background: @state-upload-bg;
}
.pagination-sm {
.pagination-sm{
height: 15px;
}
.library-paginator {
.library-paginator{
margin-left: 2em;
}
.import-type-switch{
@@ -260,3 +142,60 @@ span.expandLink {
}
}
// Follows the style for the deprecated admin libraries interface
#library-grid{
ul{
list-style: none;
}
span.expandLink{
padding-left: 12px;
display: inline-block;
vertical-align: middle;
background: url(../images/silk/resultset_next.png) no-repeat;
}
.folderRow.expanded span.expandLink{
background: url(../images/silk/resultset_bottom.png) no-repeat;
}
.folderRow span.rowIcon{
float: left;
margin-right: 5px;
width: 16px;
height: 16px;
display: inline-block;
vertical-align: middle;
background: url(../images/silk/folder.png);
}
.libraryItem-error{
margin-right: 2px;
padding: 0 2px 0 2px;
border: 1px solid @state-danger-border;
background: @state-danger-bg;
}
.libraryItem-queued{
margin-right: 2px;
padding: 0 2px 0 2px;
border: 1px solid @state-default-border;
background: @state-default-bg;
}
.libraryItem-running{
margin-right: 2px;
padding: 0 2px 0 2px;
border: 1px solid @state-running-border;
background: @state-running-bg;
}
.libraryItem-upload{
margin-right: 2px;
padding: 0 2px 0 2px;
border: 1px solid @state-upload-border;
background: @state-upload-bg;
}
}
.libraryTitle{
th{
text-align: left;
}
.rowTitle{
padding: 2px;
}
}
config/datatypes_conf.xml.sample
+2
View File
@@ -320,6 +320,7 @@
<datatype extension="sif" type="galaxy.datatypes.graph:Sif" display_in_upload="true"/>
<!-- datatypes storing triples -->
<datatype extension="triples" type="galaxy.datatypes.triples:Triples" display_in_upload="false"/>
<datatype extension="hdt" type="galaxy.datatypes.triples:HDT" display_in_upload="true"/>
<datatype extension="nt" type="galaxy.datatypes.triples:NTriples" display_in_upload="true"/>
<datatype extension="n3" type="galaxy.datatypes.triples:N3" display_in_upload="true"/>
<datatype extension="ttl" type="galaxy.datatypes.triples:Turtle" display_in_upload="true"/>
@@ -595,6 +596,7 @@
<sniffer type="galaxy.datatypes.proteomics:ThermoRAW"/>
<sniffer type="galaxy.datatypes.molecules:CML"/>
<sniffer type="galaxy.datatypes.xml:GenericXml"/>
<sniffer type="galaxy.datatypes.triples:HDT"/>
<sniffer type="galaxy.datatypes.triples:Turtle"/>
<sniffer type="galaxy.datatypes.triples:NTriples"/>
<sniffer type="galaxy.datatypes.triples:Jsonld"/>
config/galaxy.ini.sample
+12 -3
View File
@@ -125,6 +125,11 @@ paste.app_factory = galaxy.web.buildapp:app_factory
# but prefixed with install_ are also available).
#install_database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
# Setting the following option to true will cause Galaxy to automatically
# migrate the database forward after updates. This is not recommended for production
# use.
#database_auto_migrate = False
# -- Files and directories
# Dataset files are stored in this directory.
@@ -187,6 +192,10 @@ paste.app_factory = galaxy.web.buildapp:app_factory
# all Conda resolvers, but multiple resolvers can be configured independently
# in dependency_resolvers_config_file and these options overridden.
# Location on the filesystem where Conda packages are installed
# conda_prefix is the location on the filesystem where Conda packages and environments are installed
# IMPORTANT: Due to a current limitation in conda, the total length of the
# conda_prefix and the job_working_directory path should be less than 50 characters!
#conda_prefix = <tool_dependency_dir>/_conda
# Override the Conda executable to use, it will default to the one on the
# PATH (if available) and then to <conda_prefix>/bin/conda
@@ -525,13 +534,13 @@ nglims_config_file = tool-data/nglims.yaml
# The URL linked by the "How to Cite Galaxy" link in the "Help" menu.
#citation_url = https://wiki.galaxyproject.org/CitingGalaxy
#The URL linked by the "Search" link in the "Help" menu.
# The URL linked by the "Search" link in the "Help" menu.
#search_url = http://galaxyproject.org/search/usegalaxy/
#The URL linked by the "Mailing Lists" link in the "Help" menu.
# The URL linked by the "Mailing Lists" link in the "Help" menu.
#mailing_lists_url = https://wiki.galaxyproject.org/MailingLists
#The URL linked by the "Videos" link in the "Help" menu.
# The URL linked by the "Videos" link in the "Help" menu.
#screencasts_url = https://vimeo.com/galaxyproject
# The URL linked by the "Terms and Conditions" link in the "Help" menu, as well
lib/galaxy/app.py
+10 -17
View File
@@ -5,13 +5,6 @@ import sys
import time
import os
try:
from uwsgidecorators import postfork
except:
def pf_dec(func):
return func
postfork = pf_dec
from galaxy import config, jobs
import galaxy.model
import galaxy.security
@@ -31,6 +24,7 @@ from galaxy.jobs import metrics as job_metrics
from galaxy.web.proxy import ProxyManager
from galaxy.queue_worker import GalaxyQueueWorker
from galaxy.util import heartbeat
from galaxy.util.postfork import register_postfork_function
from tool_shed.galaxy_install import update_repository_manager
@@ -154,17 +148,16 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ):
fname=self.config.heartbeat_log
)
self.heartbeat.daemon = True
@postfork
def _start():
self.heartbeat.start()
if not config.process_is_uwsgi:
_start()
register_postfork_function(self.heartbeat.start)
self.sentry_client = None
if self.config.sentry_dsn:
import raven
self.sentry_client = raven.Client(self.config.sentry_dsn)
else:
self.sentry_client = None
def postfork_sentry_client():
import raven
self.sentry_client = raven.Client(self.config.sentry_dsn)
register_postfork_function(postfork_sentry_client)
# Transfer manager client
if self.config.get_bool( 'enable_beta_job_managers', False ):
from galaxy.jobs import transfer_manager
lib/galaxy/config.py
+3 -13
View File
@@ -23,22 +23,12 @@ from galaxy.exceptions import ConfigurationError
from galaxy.util import listify
from galaxy.util import string_as_bool
from galaxy.util.dbkeys import GenomeBuilds
from galaxy.util.postfork import register_postfork_function
from galaxy.web.formatting import expand_pretty_datetime_format
from .version import VERSION_MAJOR
log = logging.getLogger( __name__ )
# The uwsgi module is automatically injected by the parent uwsgi
# process and only exists that way. If anything works, this is a
# uwsgi-managed process.
try:
import uwsgi
if uwsgi.numproc:
process_is_uwsgi = True
except ImportError:
# This is not a uwsgi process, or something went horribly wrong.
process_is_uwsgi = False
def resolve_path( path, root ):
"""If 'path' is relative make absolute by prepending 'root'"""
@@ -780,7 +770,7 @@ def configure_logging( config ):
if disable_chatty_loggers:
# Turn down paste httpserver logging
if level <= logging.DEBUG:
for chatty_logger in ["paste.httpserver.ThreadPool"]:
for chatty_logger in ["paste.httpserver.ThreadPool", "routes.middleware"]:
logging.getLogger( chatty_logger ).setLevel( logging.WARN )
# Remove old handlers
@@ -801,7 +791,7 @@ def configure_logging( config ):
from raven.handlers.logging import SentryHandler
sentry_handler = SentryHandler( config.sentry_dsn )
sentry_handler.setLevel( logging.WARN )
root.addHandler( sentry_handler )
register_postfork_function(root.addHandler, sentry_handler)
class ConfiguresGalaxyMixin:
lib/galaxy/datatypes/data.py
+7 -2
View File
@@ -503,7 +503,7 @@ class Data( object ):
"""Returns ( target_ext, existing converted dataset )"""
return datatypes_registry.find_conversion_destination_for_dataset_by_extensions( dataset, accepted_formats, **kwd )
def convert_dataset(self, trans, original_dataset, target_type, return_output=False, visible=True, deps=None, set_output_history=True):
def convert_dataset(self, trans, original_dataset, target_type, return_output=False, visible=True, deps=None, set_output_history=True, target_context=None):
"""This function adds a job to the queue to convert a dataset to another type. Returns a message about success/failure."""
converter = trans.app.datatypes_registry.get_converter_by_target_type( original_dataset.ext, target_type )
@@ -518,8 +518,13 @@ class Data( object ):
params[value.name] = deps[value.name]
elif value.type == 'data':
input_name = key
# add potentially required/common internal tool parameters e.g. '__job_resource'
if target_context:
for key, value in target_context.items():
if key.startsWith( '__' ):
params[ key ] = value
params[input_name] = original_dataset
# Run converter, job is dispatched through Queue
converted_dataset = converter.execute( trans, incoming=params, set_output_hid=visible, set_output_history=set_output_history)[1]
if len(params) > 0:
lib/galaxy/datatypes/interval.py
+2
View File
@@ -402,6 +402,8 @@ class Bed( Interval ):
data_sources = { "data": "tabix", "index": "bigwig", "feature_search": "fli" }
track_type = Interval.track_type
column_names = [ 'Chrom', 'Start', 'End', 'Name', 'Score', 'Strand', 'ThickStart', 'ThickEnd', 'ItemRGB', 'BlockCount', 'BlockSizes', 'BlockStarts' ]
"""Add metadata elements"""
MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter )
MetadataElement( name="startCol", default=2, desc="Start column", param=metadata.ColumnParameter )
lib/galaxy/datatypes/triples.py
+34 -5
View File
@@ -6,11 +6,12 @@ import data
import logging
import xml
import text
import binary
log = logging.getLogger(__name__)
class Triples( data.Text ):
class Triples( data.Data ):
"""
The abstract base class for the file format that can contain triples
"""
@@ -34,7 +35,7 @@ class Triples( data.Text ):
dataset.blurb = 'file purged from disk'
class NTriples( Triples ):
class NTriples( data.Text, Triples ):
"""
The N-Triples triple data format
"""
@@ -58,7 +59,7 @@ class NTriples( Triples ):
dataset.blurb = 'file purged from disk'
class N3( Triples ):
class N3( data.Text, Triples ):
"""
The N3 triple data format
"""
@@ -81,7 +82,7 @@ class N3( Triples ):
dataset.blurb = 'file purged from disk'
class Turtle( Triples ):
class Turtle( data.Text, Triples ):
"""
The Turtle triple data format
"""
@@ -91,7 +92,10 @@ class Turtle( Triples ):
def sniff( self, filename ):
with open(filename, "r") as f:
# @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
if re.compile( r'@prefix\s+[^:]*:\s+<[^>]*>\s\.' ).search( f.readline( 1024 ) ):
line = f.readline( 1024 )
if re.compile( r'@prefix\s+[^:]*:\s+<[^>]*>\s\.' ).search( line ):
return True
if re.compile( r'@base\s+<[^>]*>\s\.' ).search( line ):
return True
return False
@@ -156,3 +160,28 @@ class Jsonld( text.Json, Triples ):
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
class HDT( binary.Binary, Triples ):
"""
The HDT triple data format
"""
edam_format = "format_2376"
file_ext = "hdt"
def sniff( self, filename ):
with open(filename, "rb") as f:
if f.read(4) == "$HDT":
return True
return False
def set_peek( self, dataset, is_multi_byte=False ):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte )
dataset.blurb = 'HDT triple data'
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
binary.Binary.register_sniffable_binary_format('HDT', 'HDT', HDT)
lib/galaxy/jobs/__init__.py
+4 -5
View File
@@ -1298,11 +1298,10 @@ class JobWrapper( object ):
dataset.set_peek( is_multi_byte=True )
else:
dataset.set_peek()
try:
# set the name if provided by the tool
dataset.name = context['name']
except:
pass
for context_key in ['name', 'info', 'dbkey']:
if context_key in context:
context_value = context[context_key]
setattr(dataset, context_key, context_value)
else:
dataset.blurb = "empty"
if dataset.ext == 'auto':
lib/galaxy/jobs/actions/post.py
-186
View File
@@ -7,34 +7,12 @@ import datetime
import logging
import socket
from json import dumps
from markupsafe import escape
from galaxy.util import send_mail
log = logging.getLogger( __name__ )
def get_form_template(action_type, title, content, help, on_output=True ):
if on_output:
form = """
if (pja.action_type == "%s"){
p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + pja.output_name + "</span><div class='toolFormTitle'> %s <br/> on " + pja.output_name + "\
<div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>";
%s
p_str += "</div><div class='toolParamHelp'>%s</div></div>";
}""" % (action_type, title, content, help)
else:
form = """
if (pja.action_type == "%s"){
p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + "</span><div class='toolFormTitle'> %s \
<div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>";
%s
p_str += "</div><div class='toolParamHelp'>%s</div></div>";
}""" % (action_type, title, content, help)
return form
class DefaultJobAction(object):
"""
Base job action.
@@ -46,10 +24,6 @@ class DefaultJobAction(object):
def execute(cls, app, sa_session, action, job, replacement_dict=None):
pass
@classmethod
def get_config_form(cls, trans):
return "<p>Default Job Action Config Form</p>"
@classmethod
def get_short_str(cls, pja):
if pja.action_arguments:
@@ -83,14 +57,6 @@ class EmailAction(DefaultJobAction):
except Exception as e:
log.error("EmailAction PJA Failed, exception: %s" % e)
@classmethod
def get_config_form(cls, trans):
form = """
p_str += "<label for='pja__"+pja.output_name+"__EmailAction'>There are no additional options for this action. You will be emailed upon job completion.</label>\
<input type='hidden' value='%s' name='pja__"+pja.output_name+"__EmailAction__host'/><input type='hidden' name='pja__"+pja.output_name+"__EmailAction'/>";
""" % trans.request.host
return get_form_template(cls.name, cls.verbose_name, form, "This action will send an email notifying you when the job is done.", on_output=False)
@classmethod
def get_short_str(cls, pja):
if pja.action_arguments and 'host' in pja.action_arguments:
@@ -109,25 +75,6 @@ class ChangeDatatypeAction(DefaultJobAction):
if action.output_name == '' or dataset_assoc.name == action.output_name:
app.datatypes_registry.change_datatype( dataset_assoc.dataset, action.action_arguments['newtype'])
@classmethod
def get_config_form(cls, trans):
dt_list = ""
dtnames = [ dtype_name for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems()]
dtnames.sort()
for dt_name in dtnames:
dt_list += """<option id='pja__"+pja.output_name+"__ChangeDatatypeAction__newtype__%s' value='%s'>%s</option>""" % (dt_name, dt_name, dt_name)
ps = """
p_str += "<label for='pja__"+pja.output_name+"__ChangeDatatypeAction__newtype'>New Datatype:</label>\
<select id='pja__"+pja.output_name+"__ChangeDatatypeAction__newtype' name='pja__"+pja.output_name+"__ChangeDatatypeAction__newtype'>\
%s\
</select>";
if (pja.action_arguments !== undefined && pja.action_arguments.newtype !== undefined){
p_str += "<scrip" + "t type='text/javascript'>$('#pja__" + pja.output_name + "__ChangeDatatypeAction__newtype').val('" + pja.action_arguments.newtype + "');</scrip" + "t>";
}
""" % dt_list
# Note the scrip + t hack above. Is there a better way?
return get_form_template(cls.name, cls.verbose_name, ps, 'This action will change the datatype of the output to the indicated value.')
@classmethod
def get_short_str(cls, pja):
return "Set the datatype of output '%s' to '%s'" % (escape(pja.output_name),
@@ -218,29 +165,6 @@ class RenameDatasetAction(DefaultJobAction):
if action.output_name == '' or dataset_assoc.name == action.output_name:
dataset_assoc.dataset.name = new_name
@classmethod
def get_config_form(cls, trans):
form = """
if (pja.action_arguments && pja.action_arguments.newname){
p_str += "<label for='pja__"+pja.output_name+"__RenameDatasetAction__newname'>New output name:</label>\
<input type='text' name='pja__"+pja.output_name+"__RenameDatasetAction__newname' value=\\"" + pja.action_arguments.newname.replace(/"/g, "&quot;") + "\\"/>";
}
else{
p_str += "<label for='pja__"+pja.output_name+"__RenameDatasetAction__newname'>New output name:</label>\
<input type='text' name='pja__"+pja.output_name+"__RenameDatasetAction__newname' value=''/>";
}
inputlist = [];
$.each(node.input_terminals, function(i, v){
inputlist.push(v.name);
});
if (inputlist !== []){
p_str += "Available inputs are: <strong>" + inputlist.join(', ') + "</strong>";
}else{
p_str += "No inputs are available for templating into this action.";
}
"""
return get_form_template(cls.name, cls.verbose_name, form, "This action will rename the result dataset. See <a href='https://wiki.galaxyproject.org/Learn/AdvancedWorkflow/Variables'>the wiki</a> for usage information.")
@classmethod
def get_short_str(cls, pja):
# Prevent renaming a dataset to the empty string.
@@ -261,14 +185,6 @@ class HideDatasetAction(DefaultJobAction):
if dataset_assoc.dataset.state != dataset_assoc.dataset.states.ERROR and ( action.output_name == '' or dataset_assoc.name == action.output_name ):
dataset_assoc.dataset.visible = False
@classmethod
def get_config_form(cls, trans):
return """
if (pja.action_type == "HideDatasetAction"){
p_str += "<input type='hidden' name='pja__"+pja.output_name+"__HideDatasetAction'/>";
}
"""
@classmethod
def get_short_str(cls, pja):
return "Hide output '%s'." % escape(pja.output_name)
@@ -285,14 +201,6 @@ class DeleteDatasetAction(DefaultJobAction):
if action.output_name == '' or dataset_assoc.name == action.output_name:
dataset_assoc.dataset.deleted = True
@classmethod
def get_config_form(cls, trans):
form = """
p_str += "<label for='pja__"+pja.output_name+"__DeleteDatasetAction'>There are no additional options for this action. This dataset will be marked deleted.</label>\
<input type='hidden' name='pja__"+pja.output_name+"__DeleteDatasetAction'/>";
"""
return get_form_template(cls.name, cls.verbose_name, form, "This action will rename the result dataset.")
@classmethod
def get_short_str(cls, pja):
return "Delete this dataset after creation."
@@ -315,36 +223,6 @@ class ColumnSetAction(DefaultJobAction):
if v != 0:
setattr(dataset_assoc.dataset.metadata, k, v)
@classmethod
def get_config_form(cls, trans):
form = """
if (pja.action_arguments !== undefined){
(pja.action_arguments.chromCol === undefined) ? chromCol = "" : chromCol=pja.action_arguments.chromCol;
(pja.action_arguments.startCol === undefined) ? startCol = "" : startCol=pja.action_arguments.startCol;
(pja.action_arguments.endCol === undefined) ? endCol = "" : endCol=pja.action_arguments.endCol;
(pja.action_arguments.strandCol === undefined) ? strandCol = "" : strandCol=pja.action_arguments.strandCol;
(pja.action_arguments.nameCol === undefined) ? nameCol = "" : nameCol=pja.action_arguments.nameCol;
}else{
chromCol = '';
startCol = '';
endCol = '';
strandCol = '';
nameCol = '';
}
p_str += "<p>Leave any of these fields blank if they do not need to be set.</p>\
<label for='pja__"+pja.output_name+"__ColumnSetAction__chromCol'>Chrom Column</label>\
<input type='text' value='" + chromCol + "' name='pja__"+pja.output_name+"__ColumnSetAction__chromCol'/>\
<label for='pja__"+pja.output_name+"__ColumnSetAction__startCol'>Start Column</label>\
<input type='text' value='" + startCol + "' name='pja__"+pja.output_name+"__ColumnSetAction__startCol'/>\
<label for='pja__"+pja.output_name+"__ColumnSetAction__endCol'>End Column</label>\
<input type='text' value='" + endCol + "' name='pja__"+pja.output_name+"__ColumnSetAction__endCol'/>\
<label for='pja__"+pja.output_name+"__ColumnSetAction__strandCol'>Strand Column</label>\
<input type='text' value='" + strandCol + "' name='pja__"+pja.output_name+"__ColumnSetAction__strandCol'/>\
<label for='pja__"+pja.output_name+"__ColumnSetAction__nameCol'>Name Column</label>\
<input type='text' value='" + nameCol + "' name='pja__"+pja.output_name+"__ColumnSetAction__nameCol'/>\";
"""
return get_form_template(cls.name, cls.verbose_name, form, "This action will set column assignments in the output dataset. Blank fields are ignored.")
@classmethod
def get_short_str(cls, pja):
return "Set the following metadata values:<br/>" + "<br/>".join(['%s : %s' % (escape(k), escape(v)) for k, v in pja.action_arguments.iteritems()])
@@ -359,33 +237,6 @@ class SetMetadataAction(DefaultJobAction):
for data in job.output_datasets:
data.set_metadata( action.action_arguments['newtype'] )
@classmethod
def get_config_form(cls, trans):
# dt_list = ""
# mdict = {}
# for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems():
# for mn, mt in dtype_value.metadata_spec.items():
# if mt.visible:
# mdict[mt.desc] = mt.param.get_html(value= mn).replace('"', "'").strip().replace('\n','')
# for k, v in mdict.items():
# dt_list += "<p><strong>" + k + ":</strong><br/>" + v + "</p>"
# form = """
# p_str += "%s";
# """ % dt_list
# return get_form_template('SetMetadataAction', 'Set Metadata', form, "This action will change metadata for the dataset.")
form = """
p_str += "<p>Leave any of these fields blank if they do not need to be set.</p><label for='pja__"+pja.output_name+"__SetMetadataAction__chromCol'>Chrom Column</label>\
<input type='text' name='pja__"+pja.output_name+"__SetMetadataAction__chromCol'/>\
<label for='pja__"+pja.output_name+"__SetMetadataAction__startCol'>Start Column</label>\
<input type='text' name='pja__"+pja.output_name+"__SetMetadataAction__startCol'/>\
<label for='pja__"+pja.output_name+"__SetMetadataAction__endCol'>End Column</label>\
<input type='text' name='pja__"+pja.output_name+"__SetMetadataAction__endCol'/>\
<label for='pja__"+pja.output_name+"__SetMetadataAction__comment_lines'>Comment Lines</label>\
<input type='text' name='pja__"+pja.output_name+"__SetMetadataAction__comment_lines'/>\
";
"""
return get_form_template(cls.name, cls.verbose_name, form, "This action will set metadata in the output dataset.")
class DeleteIntermediatesAction(DefaultJobAction):
name = "DeleteIntermediatesAction"
@@ -454,14 +305,6 @@ class DeleteIntermediatesAction(DefaultJobAction):
# We could make this work differently in the future
pass
@classmethod
def get_config_form(cls, trans):
form = """
p_str += "<label for='pja__"+pja.output_name+"__DeleteIntermediatesAction'>There are no additional options for this action.</label>\
<input type='hidden' name='pja__"+pja.output_name+"__DeleteIntermediatesAction'/>";
"""
return get_form_template(cls.name, cls.verbose_name, form, "All non-output steps of this workflow will have datasets deleted if they are no longer being used as job inputs when the job this PostJobAction is attached to is finished. You *must* be using workflow outputs (the snowflake) in your workflow for this to have any effect.", on_output=False)
@classmethod
def get_short_str(cls, pja):
return "Delete parent datasets of this step created in this workflow that aren't flagged as outputs."
@@ -481,20 +324,6 @@ class TagDatasetAction(DefaultJobAction):
app.tag_handler.set_tags_from_list( job.user, dataset_assoc.dataset, tags)
sa_session.flush()
@classmethod
def get_config_form(cls, trans):
form = """
if (pja.action_arguments && pja.action_arguments.tags){
p_str += "<label for='pja__"+pja.output_name+"__TagDatasetAction__tags'>Tags:</label>\
<input type='text' name='pja__"+pja.output_name+"__TagDatasetAction__tags' value=\\"" + pja.action_arguments.tags.replace(/"/g, "&quot;") + "\\"/>";
}
else{
p_str += "<label for='pja__"+pja.output_name+"__TagDatasetAction__tags'>Tags:</label>\
<input type='text' name='pja__"+pja.output_name+"__TagDatasetAction__tags' value=''/>";
}
"""
return get_form_template(cls.name, cls.verbose_name, form, "This action will set tags for the dataset.")
@classmethod
def get_short_str(cls, pja):
if pja.action_arguments and pja.action_arguments.get('tags', ''):
@@ -549,21 +378,6 @@ class ActionBox(object):
pass
return dumps(npd)
@classmethod
def get_add_list(cls):
addlist = "<select id='new_pja_list' name='new_pja_list'>"
for action in ActionBox.public_actions:
addlist += "<option value='%s'>%s</option>" % (ActionBox.actions[action].name, ActionBox.actions[action].verbose_name)
addlist += "</select>"
return addlist
@classmethod
def get_forms(cls, trans):
forms = ""
for action in ActionBox.actions:
forms += ActionBox.actions[action].get_config_form(trans)
return forms
@classmethod
def execute(cls, app, sa_session, pja, job, replacement_dict=None):
if pja.action_type in ActionBox.actions:
lib/galaxy/jobs/runners/slurm.py
+1 -1
View File
@@ -43,7 +43,7 @@ class SlurmJobRunner( DRMAAJobRunner ):
cmd = [ 'scontrol', '-o' ]
if '.' in ajs.job_id:
# custom slurm-drmaa-with-cluster-support job id syntax
job_id, cluster = ajs.job_id.split('.', maxsplit=1)
job_id, cluster = ajs.job_id.split('.', 1)
cmd.extend( [ '-M', cluster ] )
else:
job_id = ajs.job_id
lib/galaxy/model/__init__.py
+5 -3
View File
@@ -2034,7 +2034,7 @@ class DatasetInstance( object ):
depends_list = []
return dict([ (dep, self.get_converted_dataset(trans, dep)) for dep in depends_list ])
def get_converted_dataset(self, trans, target_ext):
def get_converted_dataset(self, trans, target_ext, target_context=None):
"""
Return converted dataset(s) if they exist, along with a dict of dependencies.
If not converted yet, do so and return None (the first time). If unconvertible, raise exception.
@@ -2073,13 +2073,15 @@ class DatasetInstance( object ):
raise NoConverterException("A dependency (%s) is missing a converter." % dependency)
except KeyError:
pass # No deps
new_dataset = next(iter(self.datatype.convert_dataset( trans, self, target_ext, return_output=True, visible=False, deps=deps, set_output_history=True ).values()))
new_dataset = next(iter(self.datatype.convert_dataset( trans, self, target_ext, return_output=True, visible=False, deps=deps, set_output_history=True, target_context=target_context ).values()))
new_dataset.hid = self.hid
new_dataset.name = self.name
assoc = ImplicitlyConvertedDatasetAssociation( parent=self, file_type=target_ext, dataset=new_dataset, metadata_safe=False )
session = trans.sa_session
session.add( new_dataset )
session.add( assoc )
session.flush()
return None
return new_dataset
def get_metadata_dataset( self, dataset_ext ):
"""
lib/galaxy/model/migrate/check.py
+1 -1
View File
@@ -89,7 +89,7 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=N
db_schema = schema.ControlledSchema( engine, migrate_repository )
if migrate_repository.versions.latest != db_schema.version:
config_arg = ''
if os.path.abspath( os.path.join( os.getcwd(), 'config', 'galaxy.ini' ) ) != galaxy_config_file:
if galaxy_config_file and os.path.abspath( os.path.join( os.getcwd(), 'config', 'galaxy.ini' ) ) != galaxy_config_file:
config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' )
raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh%s upgrade'."
% ( db_schema.version, migrate_repository.versions.latest, config_arg ) )
lib/galaxy/queues.py
+1 -1
View File
@@ -4,7 +4,7 @@ All message queues used by Galaxy
"""
from galaxy.config import process_is_uwsgi
from galaxy.util.postfork import process_is_uwsgi
from kombu import Exchange, Queue, Connection
lib/galaxy/tools/actions/__init__.py
+2 -11
View File
@@ -66,16 +66,7 @@ class DefaultToolAction( object ):
if converted_dataset:
data = converted_dataset
else:
# FIXME: merge with hda.get_converted_dataset() mode as it's nearly identical.
# run converter here
new_data = data.datatype.convert_dataset( trans, data, target_ext, return_output=True, visible=False ).values()[0]
new_data.hid = data.hid
new_data.name = data.name
trans.sa_session.add( new_data )
assoc = trans.app.model.ImplicitlyConvertedDatasetAssociation( parent=data, file_type=target_ext, dataset=new_data, metadata_safe=False )
trans.sa_session.add( assoc )
trans.sa_session.flush()
data = new_data
data = data.get_converted_dataset( trans, target_ext, target_context=parent )
if not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
raise Exception( "User does not have permission to use a dataset (%s) provided for input." % data.id )
@@ -620,7 +611,7 @@ class DefaultToolAction( object ):
"""
if output.actions:
for action in output.actions.actions:
if action.tag == "metadata":
if action.tag == "metadata" and action.default:
metadata_new_value = fill_template( action.default, context=params ).split(",")
dataset.metadata.__setattr__(str(action.name), metadata_new_value)
lib/galaxy/tools/deps/conda_util.py
+38 -1
View File
@@ -1,6 +1,7 @@
import functools
import hashlib
import json
import logging
import os
import re
import shutil
@@ -12,6 +13,8 @@ import yaml
from ..deps import commands
log = logging.getLogger(__name__)
# Not sure there are security concerns, lets just fail fast if we are going
# break shell commands we are building.
SHELL_UNSAFE_PATTERN = re.compile(r"[\s\"']")
@@ -100,6 +103,40 @@ class CondaContext(object):
else:
return None
def is_conda_installed(self):
"""
Check if conda_exec exists
"""
if os.path.exists(self.conda_exec):
return True
else:
return False
def can_install_conda(self):
"""
If conda_exec is set to a path outside of conda_prefix,
there is no use installing conda into conda_prefix, since it can't be used by galaxy.
If conda_exec equals conda_prefix/bin/conda, we can install conda if either conda_prefix
does not exist or is empty.
"""
conda_exec = os.path.abspath(self.conda_exec)
conda_prefix_plus_exec = os.path.abspath(os.path.join(self.conda_prefix, 'bin/conda'))
if conda_exec == conda_prefix_plus_exec:
if not os.path.exists(self.conda_prefix):
return True
elif os.listdir(self.conda_prefix) == []:
os.rmdir(self.conda_prefix) # Conda's install script fails if path exists (even if empty).
return True
else:
log.warning("Cannot install Conda because conda_prefix '%s' exists and is not empty.",
self.conda_prefix)
return False
else:
log.warning("Skipping installation of Conda into conda_prefix '%s', "
"since conda_exec '%s' is set to a path outside of conda_prefix.",
self.conda_prefix, self.conda_exec)
return False
def load_condarc(self):
condarc = self.condarc
if os.path.exists(condarc):
@@ -270,7 +307,7 @@ def hash_conda_packages(conda_packages, conda_target=None):
# these commands as Python
def install_conda(conda_context=None):
conda_context = _ensure_conda_context(conda_context)
f, script_path = tempfile.mkstemp(suffix=".bash", prefix="conda_install")
f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install")
os.close(f)
download_cmd = " ".join(commands.download_command(conda_link(), to=script_path, quote_url=True))
install_cmd = "bash '%s' -b -p '%s'" % (script_path, conda_context.conda_prefix)
lib/galaxy/tools/deps/containers.py
+9 -5
View File
@@ -326,12 +326,16 @@ class DockerContainer(Container):
# We have a Pulsar job directory, so everything needed (excluding index
# files) should be available in job_directory...
defaults = "$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw"
elif self.app_info.outputs_to_working_directory:
# Should need default_file_path (which is a course estimate given
# object stores anyway).
defaults = "$galaxy_root:ro,$tool_directory:ro,$job_directory:ro,$working_directory:rw,$default_file_path:ro"
else:
defaults = "$galaxy_root:ro,$tool_directory:ro,$job_directory:ro,$working_directory:rw,$default_file_path:rw"
defaults = "$galaxy_root:ro,$tool_directory:ro"
if self.job_info.job_directory:
defaults += ",$job_directory:ro"
if self.app_info.outputs_to_working_directory:
# Should need default_file_path (which is a course estimate given
# object stores anyway).
defaults += ",$working_directory:rw,$default_file_path:ro"
else:
defaults += ",$working_directory:rw,$default_file_path:rw"
if self.app_info.library_import_dir:
defaults += ",$library_import_dir:ro"
lib/galaxy/tools/deps/resolvers/conda.py
+11 -3
View File
@@ -88,11 +88,14 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
auto_install = _string_as_bool(get_option("auto_install"))
copy_dependencies = _string_as_bool(get_option("copy_dependencies"))
if not os.path.exists(conda_context.conda_prefix):
if not conda_context.is_conda_installed():
if auto_init:
if install_conda(conda_context):
if conda_context.can_install_conda():
if install_conda(conda_context):
self.disabled = True
log.warning("Conda installation requested and failed.")
else:
self.disabled = True
log.warning("Conda installation requested and failed.")
else:
self.disabled = True
log.warning("Conda not installed and auto-installation disabled.")
@@ -163,6 +166,11 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
version
)
else:
if len(conda_environment) > 79:
# TODO: remove this once conda_build version 2 is released and packages have been rebuilt.
raise Exception("Conda dependency failed to build job environment. "
"This is most likely a limitation in conda. "
"You can try to shorten the path to the job_working_directory.")
raise Exception("Conda dependency seemingly installed but failed to build job environment.")
def list_dependencies(self):
lib/galaxy/tools/parameters/output_collect.py
+2
View File
@@ -314,6 +314,8 @@ def collect_primary_datasets( tool, output, job_working_directory, input_ext, in
)
metadata_dict = new_primary_datasets_attributes.get( 'metadata', None )
if metadata_dict:
if "dbkey" in new_primary_datasets_attributes:
metadata_dict["dbkey"] = new_primary_datasets_attributes["dbkey"]
primary_data.metadata.from_JSON_dict( json_dict=metadata_dict )
else:
primary_data.set_meta()
lib/galaxy/tools/parser/xml.py
+14 -11
View File
@@ -391,15 +391,7 @@ def __parse_output_elem( output_elem ):
if name is None:
raise Exception( "Test output does not have a 'name'" )
file, attributes = __parse_test_attributes( output_elem, attrib )
primary_datasets = {}
for primary_elem in ( output_elem.findall( "discovered_dataset" ) or [] ):
primary_attrib = dict( primary_elem.attrib )
designation = primary_attrib.pop( 'designation', None )
if designation is None:
raise Exception( "Test primary dataset does not have a 'designation'" )
primary_datasets[ designation ] = __parse_test_attributes( primary_elem, primary_attrib )
attributes[ "primary_datasets" ] = primary_datasets
file, attributes = __parse_test_attributes( output_elem, attrib, parse_discovered_datasets=True )
return name, file, attributes
@@ -436,7 +428,7 @@ def __parse_element_tests( parent_element ):
return element_tests
def __parse_test_attributes( output_elem, attrib, parse_elements=False ):
def __parse_test_attributes( output_elem, attrib, parse_elements=False, parse_discovered_datasets=False ):
assert_list = __parse_assert_list( output_elem )
# Allow either file or value to specify a target file to compare result with
@@ -466,8 +458,18 @@ def __parse_test_attributes( output_elem, attrib, parse_elements=False ):
if parse_elements:
element_tests = __parse_element_tests( output_elem )
primary_datasets = {}
if parse_discovered_datasets:
for primary_elem in ( output_elem.findall( "discovered_dataset" ) or [] ):
primary_attrib = dict( primary_elem.attrib )
designation = primary_attrib.pop( 'designation', None )
if designation is None:
raise Exception( "Test primary dataset does not have a 'designation'" )
primary_datasets[ designation ] = __parse_test_attributes( primary_elem, primary_attrib )
has_checksum = md5sum or checksum
if not (assert_list or file or extra_files or metadata or has_checksum or element_tests):
has_nested_tests = extra_files or element_tests or primary_datasets
if not (assert_list or file or metadata or has_checksum or has_nested_tests):
raise Exception( "Test output defines nothing to check (e.g. must have a 'file' check against, assertions to check, metadata or checksum tests, etc...)")
attributes['assert_list'] = assert_list
attributes['extra_files'] = extra_files
@@ -475,6 +477,7 @@ def __parse_test_attributes( output_elem, attrib, parse_elements=False ):
attributes['md5'] = md5sum
attributes['checksum'] = checksum
attributes['elements'] = element_tests
attributes['primary_datasets'] = primary_datasets
return file, attributes
lib/galaxy/tools/toolbox/base.py
+2
View File
@@ -15,6 +15,7 @@ from galaxy.util import listify
from galaxy.util import parse_xml
from galaxy.util import string_as_bool
from galaxy.util.bunch import Bunch
from galaxy.util.postfork import register_postfork_function
from .parser import get_toolbox_parser, ensure_tool_conf_item
@@ -103,6 +104,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
self._init_tools_from_config( config_filename )
except:
log.exception( "Error loading tools defined in config %s", config_filename )
register_postfork_function(self._tool_conf_watcher.start)
def _init_tools_from_config( self, config_filename ):
"""
lib/galaxy/tools/toolbox/watcher.py
+4 -3
View File
@@ -14,6 +14,8 @@ except ImportError:
PollingObserver = None
can_watch = False
from galaxy.util.postfork import register_postfork_function
log = logging.getLogger( __name__ )
@@ -65,7 +67,7 @@ class ToolConfWatcher(object):
self.paths = {}
self._active = False
self._lock = threading.Lock()
self.thread = threading.Thread(target=self.check)
self.thread = threading.Thread(target=self.check, name="ToolConfWatcher.thread")
self.thread.daemon = True
self.event_handler = ToolConfFileEventHandler(reload_callback)
@@ -107,7 +109,6 @@ class ToolConfWatcher(object):
mod_time = time.ctime(os.path.getmtime(path))
with self._lock:
self.paths[path] = mod_time
self.start()
def watch_file(self, tool_conf_file):
self.monitor(tool_conf_file)
@@ -152,7 +153,7 @@ class ToolWatcher(object):
self.start()
def start(self):
self.observer.start()
register_postfork_function(self.observer.start)
def shutdown(self):
self.observer.stop()
lib/galaxy/util/postfork.py
+41
View File
@@ -0,0 +1,41 @@
"""
Handle postfork functions under uWSGI
"""
# The uwsgi module is automatically injected by the parent uwsgi
# process and only exists that way. If anything works, this is a
# uwsgi-managed process.
try:
import uwsgi
if uwsgi.numproc:
process_is_uwsgi = True
except ImportError:
# This is not a uwsgi process, or something went horribly wrong.
process_is_uwsgi = False
try:
from uwsgidecorators import postfork
except:
def pf_dec(func):
return func
postfork = pf_dec
if process_is_uwsgi:
print("WARNING: This is a uwsgi process but the uwsgidecorators library"
" is unavailable. This is likely due to using an external (not"
" in Galaxy's virtualenv) uwsgi and you may experience errors.")
postfork_functions = []
@postfork
def do_postfork():
for f, args, kwargs in [ t for t in postfork_functions ]:
f(*args, **kwargs)
def register_postfork_function(f, *args, **kwargs):
if process_is_uwsgi:
postfork_functions.append((f, args, kwargs))
else:
f(*args, **kwargs)
lib/galaxy/util/template.py
+9
View File
@@ -1,7 +1,16 @@
"""Entry point for the usage of Cheetah templating within Galaxy."""
from Cheetah.Template import Template
def fill_template( template_text, context=None, **kwargs ):
"""Fill a cheetah template out for specified context.
If template_text is None, an exception will be thrown, if context
is None (the default) - keyword arguments to this function will be used
as the context.
"""
if template_text is None:
raise TypeError("Template text specified as None to fill_template.")
if not context:
context = kwargs
return str( Template( source=template_text, searchList=[context] ) )
lib/galaxy/web/framework/middleware/sentry.py
+8 -1
View File
@@ -12,6 +12,8 @@ try:
except:
Client = None
from galaxy.util.postfork import register_postfork_function
RAVEN_IMPORT_MESSAGE = ('The Python raven package is required to use this '
'feature, please install it')
@@ -25,7 +27,12 @@ class Sentry(object):
def __init__(self, application, dsn):
assert Client is not None, RAVEN_IMPORT_MESSAGE
self.application = application
self.client = Client( dsn )
self.client = None
def postfork_sentry_client():
self.client = Client( dsn )
register_postfork_function(postfork_sentry_client)
def __call__(self, environ, start_response):
try:
lib/galaxy/web/proxy/__init__.py
+1
View File
@@ -106,6 +106,7 @@ class NodeProxyLauncher(object):
"--sessions", config.proxy_session_map,
"--ip", config.dynamic_proxy_bind_ip,
"--port", str(config.dynamic_proxy_bind_port),
"--reverseProxy",
]
if config.dynamic_proxy_debug:
args.append("--verbose")
lib/galaxy/web/proxy/js/lib/main.js
+8 -2
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env node
/*
Inspiration taken from
Inspiration taken from
https://github.com/jupyter/multiuser-server/blob/master/multiuser/js/main.js
*/
var fs = require('fs');
@@ -14,6 +14,7 @@ args
.option('--port <n>', 'Public-facing port of the proxy', parseInt)
.option('--cookie <cookiename>', 'Cookie proving authentication', 'galaxysession')
.option('--sessions <file>', 'Routes file to monitor')
.option('--reverseProxy', 'Cause the proxy to rewrite location blocks with its own port')
.option('--verbose')
args.parse(process.argv);
@@ -26,7 +27,12 @@ var sessions = mapFor(args.sessions);
var dynamic_proxy_options = {
sessionCookie: args['cookie'],
sessionMap: sessions,
verbose: args.verbose
verbose: args.verbose,
port: args['port']
}
if(args.reverseProxy){
dynamic_proxy_options.reverseProxy = true;
}
var dynamic_proxy = new DynamicProxy(dynamic_proxy_options);
lib/galaxy/web/proxy/js/lib/proxy.js
+17
View File
@@ -14,6 +14,8 @@ var DynamicProxy = function(options) {
this.sessionCookie = options.sessionCookie;
this.sessionMap = options.sessionMap;
this.debug = options.verbose;
this.reverseProxy = options.reverseProxy;
this.port = options.port;
var log_errors = function(handler) {
return function (req, res) {
@@ -87,6 +89,7 @@ DynamicProxy.prototype.findSession = function(request) {
};
DynamicProxy.prototype.handleProxyRequest = function(req, res) {
var othis = this;
var target = this.targetForRequest(req);
if(this.debug) {
console.log("PROXY " + req.method + " " + req.url + " to " + target.host + ':' + target.port);
@@ -94,7 +97,21 @@ DynamicProxy.prototype.handleProxyRequest = function(req, res) {
var origin = req.headers.origin;
this.rewriteRequest(req);
res.oldWriteHead = res.writeHead;
res.writeHead = function(statusCode, headers) {
if(othis.reverseProxy && statusCode === 302){
if(res && res._headers){
if(othis.debug){
console.log("Original Location Header: " + res._headers.location);
}
if(res._headers.location){
res._headers.location = res._headers.location.replace('http://localhost/', 'http://localhost:' + othis.port + '/');
}
if(othis.debug){
console.log("Rewritten Location Header: " + res._headers.location);
}
}
}
try {
if(origin){
res.setHeader('Access-Control-Allow-Origin', origin);
lib/galaxy/web/proxy/js/package.json
+1 -1
View File
@@ -4,7 +4,7 @@
"description": "A dynamic reverse proxy for use within Galaxy",
"main": "index.js",
"author": "John Chilton",
"license": "AFL v3",
"license": "AFL-3.0",
"readmeFilename": "README.md",
"repository": {
"type": "mercurial",
lib/galaxy/webapps/galaxy/api/history_contents.py
+1 -1
View File
@@ -312,7 +312,7 @@ class HistoryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar
return rval
def __create_dataset_collection( self, trans, history, payload, **kwd ):
source = kwd.get("source", "new_collection")
source = kwd.get( "source", payload.get( "source", "new_collection" ) )
service = trans.app.dataset_collections_service
if source == "new_collection":
create_params = api_payload_to_create_params( payload )
lib/galaxy/webapps/galaxy/buildapp.py
+6 -13
View File
@@ -4,6 +4,7 @@ Provides factory methods to assemble the Galaxy web application
import os
import sys
import threading
import atexit
try:
@@ -13,7 +14,6 @@ except:
import galaxy.app
from galaxy.config import process_is_uwsgi
import galaxy.model
import galaxy.model.mapping
import galaxy.datatypes.registry
@@ -22,6 +22,7 @@ import galaxy.web.framework.webapp
from galaxy.webapps.util import build_template_error_formatters
from galaxy import util
from galaxy.util import asbool
from galaxy.util.postfork import process_is_uwsgi, register_postfork_function
from galaxy.util.properties import load_app_properties
from paste import httpexceptions
@@ -29,15 +30,6 @@ from paste import httpexceptions
import logging
log = logging.getLogger( __name__ )
try:
from uwsgidecorators import postfork
except:
# TODO: Make this function more like flask's @before_first_request w/
# registered methods etc.
def pf_dec(func):
return func
postfork = pf_dec
class GalaxyWebApplication( galaxy.web.framework.webapp.WebApplication ):
pass
@@ -136,9 +128,11 @@ def paste_app_factory( global_conf, **kwargs ):
except:
log.exception("Unable to dispose of pooled toolshed install model database connections.")
if not process_is_uwsgi:
postfork_setup()
register_postfork_function(postfork_setup)
for th in threading.enumerate():
if th.is_alive():
log.debug("Prior to webapp return, Galaxy thread %s is alive.", th)
# Return
return webapp
@@ -159,7 +153,6 @@ def uwsgi_app_factory():
return app_factory(global_conf, **kwargs)
@postfork
def postfork_setup():
from galaxy.app import app
if process_is_uwsgi:
lib/galaxy/webapps/reports/buildapp.py
+1 -1
View File
@@ -10,8 +10,8 @@ from inspect import isclass
from paste import httpexceptions
from galaxy.config import process_is_uwsgi
from galaxy.util import asbool
from galaxy.util.postfork import process_is_uwsgi
from galaxy.webapps.util import build_template_error_formatters
import galaxy.model
lib/galaxy/webapps/tool_shed/api/repositories.py
+72
View File
@@ -717,6 +717,78 @@ class RepositoriesController( BaseAPIController ):
[ trans.security.encode_id( x.category.id ) for x in repository.categories ]
return repository_dict
@expose_api_raw_anonymous_and_sessionless
def updates( self, trans, **kwd ):
"""
GET /api/repositories/updates
Return a dictionary with boolean values for whether there are updates available
for the repository revision, newer installable revisions available,
the revision is the latest installable revision, and if the repository is deprecated.
:param owner: owner of the repository
:type owner: str
:param name: name of the repository
:type name: str
:param changeset_revision: changeset of the repository
:type changeset_revision: str
:param hexlify: flag whether to hexlify the response (for backward compatibility)
:type changeset: boolean
:returns: information about repository deprecations, updates, and upgrades
:rtype: dict
"""
name = kwd.get( 'name', None )
owner = kwd.get( 'owner', None )
changeset_revision = kwd.get( 'changeset_revision', None )
hexlify_this = util.asbool( kwd.get( 'hexlify', True ) )
repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner )
if repository:
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app,
trans.security.encode_id( repository.id ),
changeset_revision )
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
tool_shed_status_dict = {}
# Handle repository deprecation.
tool_shed_status_dict[ 'repository_deprecated' ] = str( repository.deprecated )
# Handle latest installable revision.
if changeset_revision == repository.tip( trans.app ):
tool_shed_status_dict[ 'latest_installable_revision' ] = 'True'
else:
next_installable_revision = metadata_util.get_next_downloadable_changeset_revision( repository, repo, changeset_revision )
if repository_metadata is None:
if next_installable_revision and next_installable_revision != changeset_revision:
tool_shed_status_dict[ 'latest_installable_revision' ] = 'True'
else:
tool_shed_status_dict[ 'latest_installable_revision' ] = 'False'
else:
if next_installable_revision and next_installable_revision != changeset_revision:
tool_shed_status_dict[ 'latest_installable_revision' ] = 'False'
else:
tool_shed_status_dict[ 'latest_installable_revision' ] = 'True'
# Handle revision updates.
if changeset_revision == repository.tip( trans.app ):
tool_shed_status_dict[ 'revision_update' ] = 'False'
else:
if repository_metadata is None:
tool_shed_status_dict[ 'revision_update' ] = 'True'
else:
tool_shed_status_dict[ 'revision_update' ] = 'False'
# Handle revision upgrades.
metadata_revisions = [ revision[ 1 ] for revision in metadata_util.get_metadata_revisions( repository, repo ) ]
num_metadata_revisions = len( metadata_revisions )
for index, metadata_revision in enumerate( metadata_revisions ):
if index == num_metadata_revisions:
tool_shed_status_dict[ 'revision_upgrade' ] = 'False'
break
if metadata_revision == changeset_revision:
if num_metadata_revisions - index > 1:
tool_shed_status_dict[ 'revision_upgrade' ] = 'True'
else:
tool_shed_status_dict[ 'revision_upgrade' ] = 'False'
break
return encoding_util.tool_shed_encode( tool_shed_status_dict ) if hexlify_this else json.dumps( tool_shed_status_dict )
return encoding_util.tool_shed_encode({}) if hexlify_this else json.dumps({})
@expose_api_anonymous_and_sessionless
def show_tools( self, trans, id, changeset, **kwd ):
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app,
lib/galaxy/webapps/tool_shed/app.py
+1 -1
View File
@@ -6,11 +6,11 @@ import galaxy.quota
import galaxy.tools.data
import galaxy.webapps.tool_shed.model
from galaxy import tools
from galaxy.config import configure_logging
from galaxy.managers.tags import CommunityTagManager
from galaxy.openid.providers import OpenIDProviders
from galaxy.util.dbkeys import GenomeBuilds
from galaxy.web import security
from galaxy.config import configure_logging
import tool_shed.repository_registry
import tool_shed.repository_types.registry
from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager
lib/galaxy/webapps/tool_shed/buildapp.py
+31 -1
View File
@@ -4,7 +4,9 @@ Provides factory methods to assemble the Galaxy web application
import atexit
import logging
import os
import routes
from six.moves.urllib.parse import parse_qs
from inspect import isclass
from paste import httpexceptions
from galaxy.util import asbool
@@ -14,8 +16,9 @@ import galaxy.webapps.tool_shed.model.mapping
import galaxy.web.framework.webapp
from galaxy.webapps.util import build_template_error_formatters
from galaxy import util
from galaxy.config import process_is_uwsgi
from galaxy.util.postfork import process_is_uwsgi
from galaxy.util.properties import load_app_properties
from routes.middleware import RoutesMiddleware
log = logging.getLogger( __name__ )
@@ -114,6 +117,11 @@ def app_factory( global_conf, **kwargs ):
controller='categories',
action='get_repositories',
conditions=dict( method=[ "GET" ] ) )
webapp.mapper.connect( 'show_updates_for_repository',
'/api/repositories/updates',
controller='repositories',
action='updates',
conditions=dict( method=[ "GET" ] ) )
webapp.mapper.resource( 'repository',
'repositories',
controller='repositories',
@@ -204,6 +212,12 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
# other middleware):
app = httpexceptions.make_middleware( app, conf )
log.debug( "Enabling 'httpexceptions' middleware" )
# Create a separate mapper for redirects to prevent conflicts.
redirect_mapper = routes.Mapper()
redirect_mapper = _map_redirects( redirect_mapper )
# Load the Routes middleware which we use for redirecting
app = RoutesMiddleware( app, redirect_mapper )
log.debug( "Enabling 'routes' middleware" )
# If we're using remote_user authentication, add middleware that
# protects Galaxy from improperly configured authentication in the
# upstream server
@@ -281,3 +295,19 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
def wrap_in_static( app, global_conf, **local_conf ):
urlmap, _ = galaxy.web.framework.webapp.build_url_map( app, global_conf, local_conf )
return urlmap
def _map_redirects( mapper ):
"""
Add redirect to the Routes mapper and forward the received query string.
Subsequently when the redirect is triggered in Routes middleware the request
will not even reach the webapp.
"""
def forward_qs(environ, result):
qs_dict = parse_qs(environ['QUERY_STRING'])
for qs in qs_dict:
result[ qs ] = qs_dict[ qs ]
return True
mapper.redirect( "/repository/status_for_installed_repository", "/api/repositories/updates/", _redirect_code="301 Moved Permanently", conditions=dict( function=forward_qs ) )
return mapper
lib/galaxy/webapps/tool_shed/controllers/hg.py
+13
View File
@@ -5,6 +5,7 @@ from mercurial.hgweb.request import wsgiapplication
from galaxy import web
from galaxy.web.base.controller import BaseUIController
from tool_shed.util.repository_util import get_repository_by_name_and_owner
log = logging.getLogger(__name__)
@@ -15,9 +16,21 @@ class HgController( BaseUIController ):
# The os command that results in this method being called will look something like:
# hg clone http://test@127.0.0.1:9009/repos/test/convert_characters1
hgweb_config = trans.app.hgweb_config_manager.hgweb_config
cmd = kwd.get( 'cmd', None )
def make_web_app():
hgwebapp = hgwebdir( hgweb_config )
return hgwebapp
wsgi_app = wsgiapplication( make_web_app )
if cmd == 'getbundle':
path_info = kwd.get( 'path_info', None )
if path_info:
owner, name = path_info.split( '/' )
repository = get_repository_by_name_and_owner( trans.app, name, owner )
if repository:
times_downloaded = repository.times_downloaded
times_downloaded += 1
repository.times_downloaded = times_downloaded
trans.sa_session.add( repository )
trans.sa_session.flush()
return wsgi_app
lib/galaxy/webapps/tool_shed/controllers/repository.py
-58
View File
@@ -2600,64 +2600,6 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
action='sharable_owner',
owner=owner ) )
@web.expose
def status_for_installed_repository( self, trans, **kwd ):
"""
Handle a request from a local Galaxy instance, returning a dictionary with boolean values for whether there are updates available
for the repository revision, newer installable revisions available, the revision is the latest installable revision, or if the repository
is deprecated.
"""
name = kwd.get( 'name', None )
owner = kwd.get( 'owner', None )
changeset_revision = kwd.get( 'changeset_revision', None )
repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner )
if repository:
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app,
trans.security.encode_id( repository.id ),
changeset_revision )
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
tool_shed_status_dict = {}
# Handle repository deprecation.
tool_shed_status_dict[ 'repository_deprecated' ] = str( repository.deprecated )
# Handle latest installable revision.
if changeset_revision == repository.tip( trans.app ):
tool_shed_status_dict[ 'latest_installable_revision' ] = 'True'
else:
next_installable_revision = metadata_util.get_next_downloadable_changeset_revision( repository, repo, changeset_revision )
if repository_metadata is None:
if next_installable_revision and next_installable_revision != changeset_revision:
tool_shed_status_dict[ 'latest_installable_revision' ] = 'True'
else:
tool_shed_status_dict[ 'latest_installable_revision' ] = 'False'
else:
if next_installable_revision and next_installable_revision != changeset_revision:
tool_shed_status_dict[ 'latest_installable_revision' ] = 'False'
else:
tool_shed_status_dict[ 'latest_installable_revision' ] = 'True'
# Handle revision updates.
if changeset_revision == repository.tip( trans.app ):
tool_shed_status_dict[ 'revision_update' ] = 'False'
else:
if repository_metadata is None:
tool_shed_status_dict[ 'revision_update' ] = 'True'
else:
tool_shed_status_dict[ 'revision_update' ] = 'False'
# Handle revision upgrades.
metadata_revisions = [ revision[ 1 ] for revision in metadata_util.get_metadata_revisions( repository, repo ) ]
num_metadata_revisions = len( metadata_revisions )
for index, metadata_revision in enumerate( metadata_revisions ):
if index == num_metadata_revisions:
tool_shed_status_dict[ 'revision_upgrade' ] = 'False'
break
if metadata_revision == changeset_revision:
if num_metadata_revisions - index > 1:
tool_shed_status_dict[ 'revision_upgrade' ] = 'True'
else:
tool_shed_status_dict[ 'revision_upgrade' ] = 'False'
break
return encoding_util.tool_shed_encode( tool_shed_status_dict )
return encoding_util.tool_shed_encode( {} )
@web.expose
def updated_changeset_revisions( self, trans, **kwd ):
"""
lib/galaxy/workflow/modules.py
+27 -20
View File
@@ -21,7 +21,7 @@ from galaxy.tools.parameters.basic import (
from galaxy.tools.parameters.wrapped import make_dict_copy
from galaxy.tools import DefaultToolState
from galaxy.tools import ToolInputsNotReadyException
from galaxy.util import odict, listify
from galaxy.util import odict
from galaxy.util.bunch import Bunch
from galaxy.web.framework import formbuilder
from tool_shed.util import common_util
@@ -528,9 +528,20 @@ class InputDataModule( InputModule ):
def get_data_outputs( self ):
return [ dict( name='output', extensions=['input'] ) ]
def get_runtime_inputs( self, filter_set=['data'] ):
def get_filter_set( self, connections=None ):
filter_set = []
if connections:
for oc in connections:
for ic in oc.input_step.module.get_data_inputs():
if 'extensions' in ic and ic[ 'name' ] == oc.input_name:
filter_set += ic[ 'extensions' ]
if not filter_set:
filter_set = [ 'data' ]
return ', '.join( filter_set )
def get_runtime_inputs( self, connections=None ):
label = self.state.get( "name", "Input Dataset" )
return dict( input=DataToolParameter( None, Element( "param", name="input", label=label, multiple=False, type="data", format=', '.join(filter_set) ), self.trans ) )
return dict( input=DataToolParameter( None, Element( "param", name="input", label=label, multiple=False, type="data", format=self.get_filter_set( connections ) ), self.trans ) )
class InputDataCollectionModule( InputModule ):
@@ -545,7 +556,7 @@ class InputDataCollectionModule( InputModule ):
def default_state( Class ):
return dict( name=Class.default_name, collection_type=Class.default_collection_type )
def get_runtime_inputs( self, filter_set=['data'] ):
def get_runtime_inputs( self, **kwds ):
label = self.state.get( "name", self.default_name )
collection_type = self.state.get( "collection_type", self.default_collection_type )
input_element = Element( "param", name="input", label=label, type="data_collection", collection_type=collection_type )
@@ -1141,28 +1152,24 @@ class ToolModule( WorkflowModule ):
def add_dummy_datasets( self, connections=None, steps=None ):
if connections:
# Store onnections by input name
# Store connections by input name
input_connections_by_name = dict( ( conn.input_name, conn ) for conn in connections )
else:
input_connections_by_name = {}
# Any connected input needs to have value RuntimeValue (these
# are not persisted so we need to do it every time)
# Any input needs to have value RuntimeValue or obtain the value from connected steps
def callback( input, prefixed_name, context, **kwargs ):
if isinstance( input, DataToolParameter ) or isinstance( input, DataCollectionToolParameter ):
if self.trans.workflow_building_mode is workflow_building_modes.USE_HISTORY:
if connections is None or prefixed_name in input_connections_by_name:
if steps:
connection = input_connections_by_name[ prefixed_name ]
output_step = next( output_step for output_step in steps if connection.output_step_id == output_step.id )
if output_step.type.startswith( 'data' ):
output_inputs = output_step.module.get_runtime_inputs()
output_value = output_inputs[ 'input' ].get_initial_value( self.trans, context )
if isinstance( input, DataToolParameter ):
for v in listify( output_value ):
if isinstance( v, self.trans.app.model.HistoryDatasetCollectionAssociation ):
return v.to_hda_representative()
return output_value
if connections is not None and steps is not None and self.trans.workflow_building_mode is workflow_building_modes.USE_HISTORY:
if prefixed_name in input_connections_by_name:
connection = input_connections_by_name[ prefixed_name ]
output_step = next( output_step for output_step in steps if connection.output_step_id == output_step.id )
if output_step.type.startswith( 'data' ):
output_inputs = output_step.module.get_runtime_inputs( connections=connections )
output_value = output_inputs[ 'input' ].get_initial_value( self.trans, context )
if isinstance( input, DataToolParameter ) and isinstance( output_value, self.trans.app.model.HistoryDatasetCollectionAssociation ):
return output_value.to_hda_representative()
return output_value
return RuntimeValue()
else:
return input.get_initial_value( self.trans, context )
lib/galaxy_ext/metadata/set_metadata.py
+3 -2
View File
@@ -141,10 +141,11 @@ def set_metadata():
json.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) # setting metadata has failed somehow
for i, ( filename, file_dict ) in enumerate( new_job_metadata_dict.iteritems(), start=1 ):
new_dataset = galaxy.model.Dataset( id=-i, external_filename=os.path.join( tool_job_working_directory, file_dict[ 'filename' ] ) )
new_dataset_filename = os.path.join( tool_job_working_directory, "working", file_dict[ 'filename' ] )
new_dataset = galaxy.model.Dataset( id=-i, external_filename=new_dataset_filename )
extra_files = file_dict.get( 'extra_files', None )
if extra_files is not None:
new_dataset._extra_files_path = os.path.join( tool_job_working_directory, extra_files )
new_dataset._extra_files_path = os.path.join( tool_job_working_directory, "working", extra_files )
new_dataset.state = new_dataset.states.OK
new_dataset_instance = galaxy.model.HistoryDatasetAssociation( id=-i, dataset=new_dataset, extension=file_dict.get( 'ext', 'data' ) )
set_meta_with_tool_provided( new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry )
lib/tool_shed/galaxy_install/update_repository_manager.py
+2 -1
View File
@@ -8,6 +8,7 @@ from sqlalchemy import false
import tool_shed.util.shed_util_common as suc
from galaxy import util
from galaxy.util.postfork import register_postfork_function
from tool_shed.util import common_util
from tool_shed.util import encoding_util
from tool_shed.util import repository_util
@@ -26,7 +27,7 @@ class UpdateRepositoryManager( object ):
self.sleeper = Sleeper()
self.restarter = threading.Thread( target=self.__restarter )
self.restarter.daemon = True
self.restarter.start()
register_postfork_function(self.restarter.start)
self.seconds_to_sleep = int( app.config.hours_between_check * 3600 )
def get_update_to_changeset_revision_and_ctx_rev( self, repository ):
static/scripts/bundled/analysis.bundled.js
+5 -5
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/analysis.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/libs.bundled.js
+16 -16
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/libs.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/style/blue/base.css
+1 -1
View File
File diff suppressed because one or more lines are too long
static/style/blue/library.css
+1 -1
View File
@@ -1 +1 @@
a{text-decoration:underline}label{font-weight:normal}.library_style_container{width:95%;margin:auto;margin-top:2em;overflow:auto !important}.library_style_container tr{height:32px}.library_style_container .fa{font-size:12px}.library_style_container .fa-globe{font-size:initial;margin-left:.6em}.library_style_container .libraryRow{background-color:#ebd9b2}.library_style_container .datasetHighlighted{background-color:#f9f9f9}.library_style_container .libraryItemDeleted-True{font-style:italic}.library_style_container div.libraryItemBody{padding:4px 4px 2px 4px}.library_style_container li.folderRow,.library_style_container li.datasetRow{border-top:solid 1px #c6bfa8}.library_style_container li.folderRow:hover,.library_style_container li.datasetRow:hover{background-color:#f9f9f9}.library_style_container td.right-center{vertical-align:middle !important;text-align:right}.library_style_container .library-genome-select{max-width:350px}.library_style_container .library-extension-select{max-width:140px}.library_style_container .library_table td{border-top:1px solid #5f6990 !important}.library_style_container .library_table th{border-bottom:2px solid #5f6990 !important}.library_style_container .library_table a{color:#0A143D}.library_style_container .library_table a:hover{color:maroon}.library_style_container tr.light td{background-color:white;color:black}.library_style_container tr.light:hover td{background-color:#f5e8cc}.library_style_container tr.dark td{background-color:#d6b161;color:white}.library_style_container tr.dark:hover td{background-color:#ebd4a4;color:white}.library_style_container a.dark{color:white}.library_style_container .dataset_table tr{border-bottom:1px solid #5f6990 !important}.library_style_container .dataset_table th{border:none !important}.library_style_container .dataset_table td{border:none !important}.library_style_container .dataset_table .dataset-first-column{width:30%}.library_style_container th.button_heading{width:2em}.library_style_container .bigdrop.select2-container .select2-results{max-height:300px}.library_style_container .bigdrop .select2-results{max-height:300px}.library_style_container .select2-container-multi{width:100%}.library_style_container .roles-selection{width:66%}.library_style_container #library_toolbar{margin-bottom:.5em}.library_style_container #library_toolbar span{margin-right:.2em}.library_style_container #library_toolbar .toolbar-item{margin-left:1em}.library_style_container #libraries_element button,.library_style_container #folder_items_element button{margin-left:.5em}.library_style_container img.expanderIcon{padding-right:4px}.library_style_container input.datasetCheckbox,.library_style_container li,.library_style_container ul{padding:0;margin:0}.library_style_container .rowTitle{padding:2px}.library_style_container ul{list-style:none}.library_style_container .libraryTitle th{text-align:left}.library_style_container pre.peek{background:white;color:black;overflow:auto}.library_style_container pre.peek th{color:white;background:#ebd9b2}.library_style_container .help-button{float:right}.library_style_container span.expandLink{padding-left:12px;display:inline-block;vertical-align:middle;background:url(../images/silk/resultset_next.png) no-repeat}.library_style_container .folderRow.expanded span.expandLink{background:url(../images/silk/resultset_bottom.png) no-repeat}.library_style_container .folderRow span.rowIcon{float:left;margin-right:5px;width:16px;height:16px;display:inline-block;vertical-align:middle;background:url(../images/silk/folder.png)}.library_style_container .libraryItem-error{margin-right:2px;padding:0 2px 0 2px;border:1px solid #dd1b15;background:#f9c7c5}.library_style_container .libraryItem-queued{margin-right:2px;padding:0 2px 0 2px;border:1px solid #bfbfbf;background:#eee}.library_style_container .libraryItem-running{margin-right:2px;padding:0 2px 0 2px;border:1px solid #AAAA66;background:#FFFFCC}.library_style_container .libraryItem-upload{margin-right:2px;padding:0 2px 0 2px;border:1px solid #119ac2;background:#a8e5f7}.library_style_container .pagination-sm{height:15px}.library_style_container .library-paginator{margin-left:2em}.library_style_container .import-type-switch{text-decoration:underline}.library_style_container .libimport-select-none,.library_style_container .libimport-select-all{margin-left:.5em}.library_style_container .library-modal-item{width:90%;margin-left:1em;margin-right:1em}.library_style_container .paginator-bottom{width:27em;margin-left:auto;margin-right:auto;margin-top:2em;margin-bottom:2em}
a{text-decoration:underline}label{font-weight:normal}.library_style_container{width:95%;margin:auto;margin-top:2em;overflow:auto !important}.library_style_container .fa{font-size:12px}.library_style_container .fa-globe{font-size:initial;margin-left:.6em}.library_style_container .libraryRow{background-color:#ebd9b2}.library_style_container .datasetHighlighted{background-color:#f9f9f9}.library_style_container .libraryItemDeleted-True{font-style:italic}.library_style_container div.libraryItemBody{padding:4px 4px 2px 4px}.library_style_container li.folderRow,.library_style_container li.datasetRow{border-top:solid 1px #c6bfa8}.library_style_container li.folderRow:hover,.library_style_container li.datasetRow:hover{background-color:#f9f9f9}.library_style_container td.right-center{vertical-align:middle !important;text-align:right}.library_style_container .library-genome-select{max-width:350px}.library_style_container .library-extension-select{max-width:140px}.library_style_container .library_table td{border-top:1px solid #5f6990 !important}.library_style_container .library_table th{border-bottom:2px solid #5f6990 !important}.library_style_container .library_table a{color:#0A143D}.library_style_container .library_table a:hover{color:maroon}.library_style_container tr.light td{background-color:white;color:black}.library_style_container tr.light:hover td{background-color:#f5e8cc}.library_style_container tr.dark td{background-color:#d6b161;color:white}.library_style_container tr.dark:hover td{background-color:#ebd4a4;color:white}.library_style_container a.dark{color:white}.library_style_container .dataset_table tr,.library_style_container .dataset_table th,.library_style_container .dataset_table td{border:none}.library_style_container .dataset_table .dataset-first-column{width:30%}.library_style_container th.button_heading{width:2em}.library_style_container .bigdrop.select2-container .select2-results{max-height:300px}.library_style_container .bigdrop .select2-results{max-height:300px}.library_style_container .select2-container-multi{width:100%}.library_style_container .roles-selection{width:66%}.library_style_container #library_toolbar{margin-bottom:.5em}.library_style_container #library_toolbar span{margin-right:.2em}.library_style_container #library_toolbar .toolbar-item{margin-left:1em}.library_style_container #libraries_element button,.library_style_container #folder_items_element button{margin-left:.5em}.library_style_container .help-button{float:right}.library_style_container .pagination-sm{height:15px}.library_style_container .library-paginator{margin-left:2em}.library_style_container .import-type-switch{text-decoration:underline}.library_style_container .libimport-select-none,.library_style_container .libimport-select-all{margin-left:.5em}.library_style_container .library-modal-item{width:90%;margin-left:1em;margin-right:1em}.library_style_container .paginator-bottom{width:27em;margin-left:auto;margin-right:auto;margin-top:2em;margin-bottom:2em}#library-grid ul{list-style:none}#library-grid span.expandLink{padding-left:12px;display:inline-block;vertical-align:middle;background:url(../images/silk/resultset_next.png) no-repeat}#library-grid .folderRow.expanded span.expandLink{background:url(../images/silk/resultset_bottom.png) no-repeat}#library-grid .folderRow span.rowIcon{float:left;margin-right:5px;width:16px;height:16px;display:inline-block;vertical-align:middle;background:url(../images/silk/folder.png)}#library-grid .libraryItem-error{margin-right:2px;padding:0 2px 0 2px;border:1px solid #dd1b15;background:#f9c7c5}#library-grid .libraryItem-queued{margin-right:2px;padding:0 2px 0 2px;border:1px solid #bfbfbf;background:#eee}#library-grid .libraryItem-running{margin-right:2px;padding:0 2px 0 2px;border:1px solid #AAAA66;background:#FFFFCC}#library-grid .libraryItem-upload{margin-right:2px;padding:0 2px 0 2px;border:1px solid #119ac2;background:#a8e5f7}.libraryTitle th{text-align:left}.libraryTitle .rowTitle{padding:2px}
templates/webapps/galaxy/workflow/run.mako
+2 -19
View File
@@ -23,14 +23,7 @@
'action_arguments' : pja.action_arguments
} for pja in step.post_job_actions ]
else:
type_filter = []
for oc in step.output_connections:
for ic in oc.input_step.module.get_data_inputs():
if 'extensions' in ic and ic[ 'name' ] == oc.input_name:
type_filter += ic[ 'extensions' ]
if not type_filter:
type_filter = [ 'data' ]
inputs = step.module.get_runtime_inputs( filter_set=type_filter )
inputs = step.module.get_runtime_inputs( connections=step.output_connections )
step_model = {
'name' : step.module.name,
'inputs' : [ input.to_dict( trans ) for input in inputs.itervalues() ]
@@ -706,17 +699,7 @@ import base64
% endif
</div>
<div class="toolFormBody">
<%
# Filter possible inputs to data types that are valid for subsequent steps
type_filter = []
for oc in step.output_connections:
for ic in oc.input_step.module.get_data_inputs():
if 'extensions' in ic and ic['name'] == oc.input_name:
type_filter += ic['extensions']
if not type_filter:
type_filter = ['data']
%>
${do_inputs( module.get_runtime_inputs(filter_set=type_filter), step.state.inputs, errors.get( step.id, dict() ), "", step, None, used_accumulator )}
${do_inputs( module.get_runtime_inputs( connections=step.output_connections ), step.state.inputs, errors.get( step.id, dict() ), "", step, None, used_accumulator )}
</div>
</div>
%endif
test/base/interactor.py
+26 -7
View File
@@ -7,6 +7,7 @@ from logging import getLogger
from requests import get, post, delete, patch
from six import StringIO
from six import text_type
from galaxy import util
from galaxy.tools.parser.interface import TestCollectionDef
@@ -87,27 +88,45 @@ class GalaxyInteractorApi( object ):
def verify_output_dataset( self, history_id, hda_id, outfile, attributes, shed_tool_id ):
fetcher = self.__dataset_fetcher( history_id )
self.twill_test_case.verify_hid( outfile, hda_id=hda_id, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id )
self.twill_test_case.verify_hid(
outfile,
hda_id=hda_id,
attributes=attributes,
dataset_fetcher=fetcher,
shed_tool_id=shed_tool_id
)
self._verify_metadata( history_id, hda_id, attributes )
def _verify_metadata( self, history_id, hid, attributes ):
"""Check dataset metadata.
ftype on output maps to `file_ext` on the hda's API description, `name`, `info`,
and `dbkey` all map to the API description directly. Other metadata attributes
are assumed to be datatype-specific and mapped with a prefix of `metadata_`.
"""
metadata = attributes.get( 'metadata', {} ).copy()
for key, value in metadata.copy().items():
new_key = "metadata_%s" % key
metadata[ new_key ] = metadata[ key ]
del metadata[ key ]
if key not in ['name', 'info']:
new_key = "metadata_%s" % key
metadata[ new_key ] = metadata[ key ]
del metadata[ key ]
elif key == "info":
metadata[ "misc_info" ] = metadata[ "info" ]
del metadata[ "info" ]
expected_file_type = attributes.get( 'ftype', None )
if expected_file_type:
metadata[ "file_ext" ] = expected_file_type
if metadata:
import time
time.sleep(5)
dataset = self._get( "histories/%s/contents/%s" % ( history_id, hid ) ).json()
for key, value in metadata.items():
try:
dataset_value = dataset.get( key, None )
if dataset_value != value:
msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]."
msg_params = ( key, value, dataset_value )
if text_type(dataset_value) != text_type(value):
msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]. Dataset API value was [%s]."
msg_params = ( key, value, dataset_value, dataset )
msg = msg % msg_params
raise Exception( msg )
except KeyError:
test/functional/tools/dbkey_output_action.xml
+36
View File
@@ -0,0 +1,36 @@
<tool id="dbkey_output_action" name="dbkey_output_action" version="0.1.0">
<command>echo foo > $mapped_reads</command>
<inputs>
<param name="input" type="data" />
<param name="index" type="select" label="Using reference genome">
<options from_data_table="test_fasta_indexes">
<filter type="data_meta" ref="input" key="dbkey" column="1" />
<validator type="no_options" message="No reference genome is available for the build associated with the selected input dataset" />
</options>
</param>
</inputs>
<outputs>
<data format="txt" name="mapped_reads">
<actions>
<action type="metadata" name="dbkey">
<option type="from_data_table" name="test_fasta_indexes" column="1" offset="0">
<filter type="param_value" column="0" value="#" compare="startswith" keep="False"/>
<filter type="param_value" ref="index" column="0"/>
</option>
</action>
</actions>
</data>
</outputs>
<tests>
<test>
<param name="input" value="simple_line.txt" dbkey="hg18" />
<param name="index" value="hg18"/>
<output name="mapped_reads">
<metadata name="dbkey" value="hg18" />
<assert_contents>
<has_text text="foo" />
</assert_contents>
</output>
</test>
</tests>
</tool>
test/functional/tools/samples_tool_conf.xml
+4
View File
@@ -15,9 +15,13 @@
<tool file="multi_output.xml" />
<tool file="multi_output_configured.xml" />
<tool file="multi_output_assign_primary.xml" />
<tool file="tool_provided_metadata_1.xml" />
<tool file="tool_provided_metadata_2.xml" />
<tool file="tool_provided_metadata_3.xml" />
<tool file="inputs_as_json.xml" />
<tool file="dbkey_filter_input.xml" />
<tool file="dbkey_filter_multi_input.xml" />
<tool file="dbkey_output_action.xml" />
<tool file="composite_output.xml" />
<tool file="composite_output_tests.xml" />
<tool file="unicode_stream.xml" />
test/functional/tools/tool_provided_metadata_1.xml
+29
View File
@@ -0,0 +1,29 @@
<tool id="tool_provided_metadata_1" name="tool_provided_metadata_1">
<command>
echo "This is a line of text." > $out1;
cp $c1 galaxy.json;
</command>
<configfiles>
<configfile name="c1">{"type": "dataset", "dataset_id": $out1.dataset.dataset.id, "name": "my dynamic name", "ext": "txt", "info": "my dynamic info", "dbkey": "cust1"}</configfile>
</configfiles>
<inputs>
<param name="input1" type="data" label="Input Dataset"/>
</inputs>
<outputs>
<!-- Set format="auto" to read from galaxy.json, use auto_format="true"
to sniff. -->
<data name="out1" format="auto" />
</outputs>
<help>
</help>
<tests>
<test>
<param name="input1" value="simple_line.txt" />
<output name="out1" file="simple_line.txt" ftype="txt">
<metadata name="name" value="my dynamic name" />
<metadata name="info" value="my dynamic info" />
<metadata name="dbkey" value="cust1" />
</output>
</test>
</tests>
</tool>
test/functional/tools/tool_provided_metadata_2.xml
+38
View File
@@ -0,0 +1,38 @@
<tool id="tool_provided_metadata_2" name="tool_provided_metadata_2">
<command>
echo "1" > sample1.report.tsv;
echo "2" > sample2.report.tsv;
cp $c1 galaxy.json;
</command>
<configfiles>
<configfile name="c1">{"type": "new_primary_dataset", "filename": "sample1.report.tsv", "name": "cool name 1", "ext": "txt", "info": "cool 1 info", "dbkey": "hg19"}
{"type": "new_primary_dataset", "filename": "sample2.report.tsv", "name": "cool name 2", "ext": "txt", "info": "cool 2 info", "dbkey": "hg19"}
</configfile>
</configfiles>
<inputs>
<param name="input" type="data" />
</inputs>
<outputs>
<data name="sample">
<discover_datasets pattern="(?P&lt;designation&gt;.+)\.report\.tsv" visible="true" />
</data>
</outputs>
<tests>
<test>
<param name="input" ftype="txt" value="simple_line.txt"/>
<output name="sample">
<discovered_dataset designation="sample1" ftype="txt">
<assert_contents><has_line line="1" /></assert_contents>
<metadata name="name" value="cool name 1" />
<metadata name="dbkey" value="hg19" />
<metadata name="info" value="cool 1 info" />
</discovered_dataset>
<discovered_dataset designation="sample2" ftype="txt">
<assert_contents><has_line line="2" /></assert_contents>
<metadata name="name" value="cool name 2" />
<metadata name="info" value="cool 2 info" />
</discovered_dataset>
</output>
</test>
</tests>
</tool>
test/functional/tools/tool_provided_metadata_3.xml
+43
View File
@@ -0,0 +1,43 @@
<tool id="tool_provided_metadata_3" name="tool_provided_metadata_3">
<command>
echo "1" > sample1.report.tsv;
echo "2" > sample2.report.tsv;
cp $c1 galaxy.json;
</command>
<configfiles>
<configfile name="c1">{"type": "new_primary_dataset", "filename": "sample1.report.tsv", "name": "cool name 1", "ext": "txt", "info": "cool 1 info", "dbkey": "hg19", "metadata": {"data_lines": 10, "foo": "bar"}}
{"type": "new_primary_dataset", "filename": "sample2.report.tsv", "name": "cool name 2", "ext": "txt", "info": "cool 2 info", "dbkey": "hg19", "metadata": {"data_lines": 20, "foo": "bar"}}
</configfile>
</configfiles>
<inputs>
<param name="input" type="data" />
</inputs>
<outputs>
<data name="sample">
<discover_datasets pattern="(?P&lt;designation&gt;.+)\.report\.tsv" visible="true" />
</data>
</outputs>
<tests>
<test>
<param name="input" ftype="txt" value="simple_line.txt"/>
<output name="sample">
<discovered_dataset designation="sample1" ftype="txt">
<assert_contents><has_line line="1" /></assert_contents>
<!-- Datatype defined metadata can be overridden/specified directly.
-->
<metadata name="data_lines" value="10" />
<!-- Non-datatype defined metadata values are ignored by the framework.
Uncommenting the following test will break this test.
-->
<!--
<metadata name="foo" value="bar" />
-->
</discovered_dataset>
<discovered_dataset designation="sample2" ftype="txt">
<assert_contents><has_line line="2" /></assert_contents>
<metadata name="data_lines" value="20" />
</discovered_dataset>
</output>
</test>
</tests>
</tool>