opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release_15.05'

Browse Source
This commit is contained in:
Dannon Baker
2015-07-01 14:29:07 -04:00
parent 6f53676f48 cf51e0b89b
commit f6b894a629
11 changed files with 76 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/plugins/visualizations/charts/static/build-app.js
+1 -6
View File
File diff suppressed because one or more lines are too long
config/plugins/visualizations/charts/static/charts/nvd3/histogram/config.js
+4 -3
View File
@@ -1,6 +1,7 @@
define(['plugin/charts/nvd3/common/config'], function(nvd3_config) {
return $.extend(true, {}, nvd3_config, {
define(['plugin/charts/forms/default'], function(config_default) {
return $.extend(true, {}, config_default, {
library : 'NVD3',
tag : 'svg',
title : 'Histogram',
category : 'Data processing (requires \'charts\' tool from Toolshed)',
execute : 'histogram',
config/plugins/visualizations/charts/static/charts/nvd3/histogram_discrete/config.js
+4 -3
View File
@@ -1,6 +1,7 @@
define(['plugin/charts/nvd3/common/config'], function(nvd3_config) {
return $.extend(true, {}, nvd3_config, {
define(['plugin/charts/forms/default'], function(config_default) {
return $.extend(true, {}, config_default, {
library : 'NVD3',
tag : 'svg',
title : 'Discrete Histogram',
category : 'Data processing (requires \'charts\' tool from Toolshed)',
execute : 'histogramdiscrete',
config/plugins/visualizations/charts/static/views/editor.js
+2 -2
View File
@@ -211,7 +211,7 @@ return Backbone.View.extend({
// add group model
_addGroupModel: function() {
var group = new Group({
id : Utils.uuid()
id : Utils.uid()
});
this.chart.groups.add(group);
return group;
@@ -260,7 +260,7 @@ return Backbone.View.extend({
// reset
_resetChart: function() {
// reset chart details
this.chart.set('id', Utils.uuid());
this.chart.set('id', Utils.uid());
this.chart.set('type', 'nvd3_bar');
this.chart.set('dataset_id', this.app.options.config.dataset_id);
this.chart.set('title', 'New Chart');
config/plugins/visualizations/charts/static/views/viewport.js
+1 -1
View File
@@ -295,7 +295,7 @@ return Backbone.View.extend({
_templateContainer: function(tag, width) {
return '<div class="charts-viewport-container" style="width:' + width + '%;">' +
'<div id="menu"/>' +
'<' + tag + ' id="' + Utils.uuid() + '" class="charts-viewport-canvas">' +
'<' + tag + ' id="' + Utils.uid() + '" class="charts-viewport-canvas">' +
'</div>';
}
lib/galaxy/datatypes/metadata.py
+11 -3
View File
@@ -122,6 +122,12 @@ class MetadataCollection( object ):
else:
self.parent._metadata[name] = value
def remove_key( self, name ):
if name in self.parent._metadata:
del self.parent._metadata[name]
else:
log.info( "Attempted to delete invalid key '%s' from MetadataCollection" % name )
def element_is_set( self, name ):
return bool( self.parent._metadata.get( name, False ) )
@@ -720,7 +726,8 @@ class JobExternalOutputMetadataWrapper( object ):
output_fnames=None, config_root=None,
config_file=None, datatypes_config=None,
job_metadata=None, compute_tmp_dir=None,
include_command=True, kwds=None ):
include_command=True, max_metadata_value_size=0,
kwds=None):
kwds = kwds or {}
if tmp_dir is None:
tmp_dir = MetadataTempFile.tmp_dir
@@ -819,9 +826,10 @@ class JobExternalOutputMetadataWrapper( object ):
sa_session.add( metadata_files )
sa_session.flush()
metadata_files_list.append( metadata_files )
args = "%s %s %s" % ( datatypes_config,
args = "%s %s %s %s" % ( datatypes_config,
job_metadata,
" ".join( map( __metadata_files_list_to_cmd_line, metadata_files_list ) ) )
" ".join( map( __metadata_files_list_to_cmd_line, metadata_files_list ) ),
max_metadata_value_size)
if include_command:
# return command required to build
fd, fp = tempfile.mkstemp( suffix='.py', dir=tmp_dir, prefix="set_metadata_" )
lib/galaxy/jobs/__init__.py
+1
View File
@@ -1596,6 +1596,7 @@ class JobWrapper( object ):
config_file=config_file,
datatypes_config=datatypes_config,
job_metadata=os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE ),
max_metadata_value_size=self.app.config.max_metadata_value_size,
**kwds )
@property
lib/galaxy/tools/__init__.py
+10
View File
@@ -500,6 +500,16 @@ class Tool( object, Dictifiable ):
def produces_collections( self ):
return any( o.collection for o in self.outputs.values() )
@property
def produces_collections_with_unknown_structure( self ):
def output_is_dynamic(output):
if not output.collection:
return False
return output.dynamic_structure
return any( map( output_is_dynamic, self.outputs.values() ) )
def __get_job_tool_configuration(self, job_params=None):
"""Generalized method for getting this tool's job configuration.
lib/galaxy/tools/actions/metadata.py
+11 -10
View File
@@ -75,16 +75,17 @@ class SetMetadataToolAction( ToolAction ):
external_metadata_wrapper = JobExternalOutputMetadataWrapper( job )
cmd_line = external_metadata_wrapper.setup_external_metadata( dataset,
sa_session,
exec_dir = None,
tmp_dir = job_working_dir,
dataset_files_path = app.model.Dataset.file_path,
output_fnames = input_paths,
config_root = app.config.root,
config_file = app.config.config_file,
datatypes_config = app.datatypes_registry.integrated_datatypes_configs,
job_metadata = None,
include_command = False,
kwds = { 'overwrite' : overwrite } )
exec_dir=None,
tmp_dir=job_working_dir,
dataset_files_path=app.model.Dataset.file_path,
output_fnames=input_paths,
config_root=app.config.root,
config_file=app.config.config_file,
datatypes_config=app.datatypes_registry.integrated_datatypes_configs,
job_metadata=None,
include_command=False,
max_metadata_value_size=app.config.max_metadata_value_size,
kwds={ 'overwrite' : overwrite } )
incoming[ '__SET_EXTERNAL_METADATA_COMMAND_LINE__' ] = cmd_line
for name, value in tool.params_to_strings( incoming, app ).iteritems():
job.add_parameter( name, value )
lib/galaxy/workflow/run.py
+13 -1
View File
@@ -33,6 +33,12 @@ BASIC_WORKFLOW_STEP_TYPES = [ None, "tool", "data_input", "data_collection_input
def force_queue( trans, workflow ):
# Default behavior is still to just schedule workflows completley right
# away. This can be modified here in various ways.
# TODO: check for implicit connections - these should also force backgrounding
# this would fix running Dan's data manager workflows via UI.
# TODO: ensure state if populated before calling force_queue from old API
# workflow endpoint so the has_module check below is unneeded and these
# interesting workflows will work with the older endpoint.
config = trans.app.config
force_for_collection = config.force_beta_workflow_scheduled_for_collections
force_min_steps = config.force_beta_workflow_scheduled_min_steps
@@ -42,6 +48,10 @@ def force_queue( trans, workflow ):
log.info("Workflow has many steps %d, backgrounding execution" % step_count)
return True
for step in workflow.steps:
# State and module haven't been populated if workflow submitted via
# the API. API requests for "interesting" workflows should use newer
# endpoint that skips this check entirely - POST /api/workflows/<id>/invocations
has_module = hasattr(step, "module")
if step.type not in BASIC_WORKFLOW_STEP_TYPES:
log.info("Found non-basic workflow step type - backgrounding execution")
# Force all new beta modules types to be use force queueing of
@@ -50,7 +60,9 @@ def force_queue( trans, workflow ):
if step.type == "data_collection_input" and force_for_collection:
log.info("Found collection input step - backgrounding execution")
return True
if step.type == "tool" and has_module and step.module.tool.produces_collections_with_unknown_structure:
log.info("Found dynamically structured output collection - backgrounding execution")
return True
return False
lib/galaxy_ext/metadata/set_metadata.py
+18 -1
View File
@@ -3,7 +3,7 @@ Execute an external process to set_meta() on a provided list of pickled datasets
This was formerly scripts/set_metadata.py and expects these arguments:
%prog datatypes_conf.xml job_metadata_file metadata_in,metadata_kwds,metadata_out,metadata_results_code,output_filename_override,metadata_override...
%prog datatypes_conf.xml job_metadata_file metadata_in,metadata_kwds,metadata_out,metadata_results_code,output_filename_override,metadata_override... max_metadata_value_size
Galaxy should be importable on sys.path and output_filename_override should be
set to the path of the dataset on which metadata is being set
@@ -30,6 +30,8 @@ sys.path.insert( 0, new_path )
from galaxy import eggs
import pkg_resources
import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers
from galaxy.model.custom_types import total_size
galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here
from galaxy.util import stringify_dictionary_keys
from sqlalchemy.orm import clear_mappers
@@ -68,6 +70,16 @@ def set_metadata():
galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir))
galaxy.datatypes.metadata.MetadataTempFile.tmp_dir = tool_job_working_directory = os.path.abspath(os.getcwd())
# This is ugly, but to transition from existing jobs without this parameter
# to ones with, smoothly, it has to be the last optional parameter and we
# have to sniff it.
try:
max_metadata_value_size = int(sys.argv[-1])
sys.argv = sys.argv[:-1]
except ValueError:
max_metadata_value_size = 0
# max_metadata_value_size is unspecified and should be 0
# Set up datatypes registry
datatypes_config = sys.argv.pop( 1 )
datatypes_registry = galaxy.datatypes.registry.Registry()
@@ -119,6 +131,11 @@ def set_metadata():
setattr( dataset.metadata, metadata_name, metadata_file_override )
file_dict = existing_job_metadata_dict.get( dataset.dataset.id, {} )
set_meta_with_tool_provided( dataset, file_dict, set_meta_kwds, datatypes_registry )
if max_metadata_value_size:
for k, v in dataset.metadata.items():
if total_size(v) > max_metadata_value_size:
log.info("Key %s too large for metadata, discarding" % k)
dataset.metadata.remove_key(k)
dataset.metadata.to_JSON_dict( filename_out ) # write out results of set_meta
json.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) # setting metadata has succeeded
except Exception, e: