opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into maps_for_developer_only

Browse Source
This commit is contained in:
Dannon Baker
2017-11-29 12:54:04 -05:00
parent a36542a72b baba3417d0
commit 0417699825
122 changed files with 4560 additions and 1024 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
-2
View File
@@ -16,7 +16,6 @@ env:
matrix:
allow_failures:
- env: TOX_ENV=check-python-dependencies
- env: TOX_ENV=qunit
addons:
apt:
@@ -25,7 +24,6 @@ addons:
install:
- pip install tox
- if [ "$TOX_ENV" == "qunit" ]; then bash -c 'cd test/qunit && npm install'; fi
- if [ "$TOX_ENV" == "first_startup" ]; then bash -c "bash scripts/common_startup.sh && wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite && mv db_gx_rev_0127.sqlite database/universe.sqlite && bash manage_db.sh -c ./config/galaxy.ini.sample upgrade"; fi
script: tox -e $TOX_ENV
Makefile
+4 -1
View File
@@ -141,9 +141,12 @@ client-watch: node-deps ## A useful target for parallel development building.
cd client && yarn run watch
@echo "Remember to 'make client' when finished developing!"
client-test:
client-test: client ## Run qunit tests via Karma
cd client && yarn run test
client-test-watch: client ## Watch and run qunit tests on changes via Karma
cd client && yarn run test-watch
charts: node-deps ## Rebuild charts
cd client && yarn run build-charts
client/galaxy/scripts/apps/analysis.js
+22 -3
View File
@@ -13,6 +13,7 @@ import Tours from "mvc/tours";
import GridView from "mvc/grid/grid-view";
import GridShared from "mvc/grid/grid-shared";
import Workflows from "mvc/workflow/workflow";
import HistoryImport from "components/history-import.vue";
import HistoryList from "mvc/history/history-list";
import ToolFormComposite from "mvc/tool/tool-form-composite";
import QueryStringParsing from "utils/query-string-parsing";
@@ -46,20 +47,22 @@ window.app = function app(options, bootstrapped) {
"(/)tours(/)(:tour_id)": "show_tours",
"(/)user(/)": "show_user",
"(/)user(/)(:form_id)": "show_user_form",
"(/)workflow(/)": "show_workflows",
"(/)workflow/run(/)": "show_run",
"(/)pages(/)create(/)": "show_pages_create",
"(/)pages(/)edit(/)": "show_pages_edit",
"(/)pages(/)(:action_id)": "show_pages",
"(/)visualizations(/)edit(/)": "show_visualizations_edit",
"(/)visualizations/(:action_id)": "show_visualizations",
"(/)workflows/import_workflow": "show_import_workflow",
"(/)workflows/run(/)": "show_run",
"(/)workflows(/)list": "show_workflows",
"(/)workflows/list_published(/)": "show_workflows_published",
"(/)workflows/create(/)": "show_workflows_create",
"(/)histories(/)citations(/)": "show_history_citations",
"(/)histories(/)rename(/)": "show_histories_rename",
"(/)histories(/)import(/)": "show_histories_import",
"(/)histories(/)permissions(/)": "show_histories_permissions",
"(/)histories(/)(:action_id)": "show_histories",
"(/)datasets(/)list(/)": "show_datasets",
"(/)workflow/import_workflow": "show_import_workflow",
"(/)custom_builds": "show_custom_builds",
"(/)datasets/edit": "show_dataset_edit_attributes",
"(/)datasets/error": "show_dataset_error"
@@ -138,6 +141,13 @@ window.app = function app(options, bootstrapped) {
);
},
show_histories_import: function() {
var historyImportInstance = Vue.extend(HistoryImport);
var vm = document.createElement("div");
this.page.display(vm);
new historyImportInstance().$mount(vm);
},
show_histories_permissions: function() {
this.page.display(
new FormWrapper.View({
@@ -188,6 +198,15 @@ window.app = function app(options, bootstrapped) {
this.page.display(new Workflows.View());
},
show_workflows_create: function() {
this.page.display(
new FormWrapper.View({
url: `workflow/create`,
redirect: "workflow/editor"
})
);
},
show_run: function() {
this._loadWorkflow();
},
client/galaxy/scripts/apps/panels/tool-panel.js
+1 -1
View File
@@ -66,7 +66,7 @@ var ToolPanel = Backbone.View.extend({
self.$("#internal-workflows").append(
self._templateWorkflowLink({
title: menu_entry.stored_workflow.name,
href: `workflow/run?id=${menu_entry.encoded_stored_workflow_id}`
href: `workflows/run?id=${menu_entry.encoded_stored_workflow_id}`
})
);
});
client/galaxy/scripts/components/history-import.vue
+63
View File
@@ -0,0 +1,63 @@
<template>
<div class="ui-portlet-limited">
<div class="portlet-header">
<div class="portlet-title">
<i class="portlet-title-icon fa fa-upload"></i>
<span class="portlet-title-text"><b>Import a History from an Archive</b></span>
</div>
</div>
<div class="portlet-content">
<div v-if="errormessage" class="ui-message alert alert-danger">
{{ errormessage }}
</div>
<div class="portlet-body">
<form ref="form">
<div class="ui-form-element">
<div class="ui-form-title">Archived History URL</div>
<input class="ui-input" type="text" name="archive_source"/>
</div>
<div class="ui-form-element">
<div class="ui-form-title">Archived History file</div>
<input type="file" name="archive_file"/>
</div>
</form>
</div>
<div class="portlet-buttons">
<input class="btn btn-primary" type="button" value="Import History" @click="submit"/>
</div>
</div>
</div>
</template>
<script>
export default {
data() {
return {
errormessage: null
}
},
methods: {
submit: function() {
$.ajax({
url: `${Galaxy.root}api/histories`,
data: new FormData(this.$refs.form),
cache: false,
contentType: false,
processData: false,
method: "POST"
})
.done(response => {
window.location = `${Galaxy.root}histories/list?message=${response.message}&status=success`
})
.fail(response => {
let message = response.responseJSON && response.responseJSON.err_msg;
this.errormessage = message || "Import failed for unkown reason.";
});
}
}
};
</script>
<style>
.ui-message {
display: block;
}
</style>
client/galaxy/scripts/layout/menu.js
+1 -1
View File
@@ -39,7 +39,7 @@ var Collection = Backbone.Collection.extend({
title: _l("Workflow"),
tooltip: _l("Chain tools into workflows"),
disabled: !Galaxy.user.id,
url: "workflow"
url: "workflows/list"
});
//
client/galaxy/scripts/mvc/form/form-wrapper.js
+11 -6
View File
@@ -58,13 +58,18 @@ var View = Backbone.View.extend({
contentType: "application/json"
})
.done(response => {
var success_message = {
message: response.message,
status: "success",
persistent: false
};
var params = {};
if (response.id) {
params.id = response.id;
} else {
params = {
message: response.message,
status: "success",
persistent: false
}
}
if (self.redirect) {
window.location = `${Galaxy.root + self.redirect}?${$.param(success_message)}`;
window.location = `${Galaxy.root + self.redirect}?${$.param(params)}`;
} else {
form.data.matchModel(response, (input, input_id) => {
form.field_list[input_id].value(input.value);
client/galaxy/scripts/mvc/history/options-menu.js
+2 -1
View File
@@ -169,7 +169,8 @@ var menu = [
},
{
html: _l("Import from File"),
href: "history/import_archive"
href: "histories/import",
target: "_top"
}
];
client/galaxy/scripts/mvc/tag.js
+1 -1
View File
@@ -103,7 +103,7 @@ var TagsEditor = Backbone.View.extend(baseMVC.LoggableMixin)
_renderTags: function() {
var tags = this.model.get("tags");
var addButton = "static/images/fugue/tag--plus.png";
var addButton = `${Galaxy.root}static/images/fugue/tag--plus.png`;
var renderedArray = [];
_.each(tags, tag => {
tag = tag.indexOf("name:") == 0 ? tag.slice(5) : tag;
client/galaxy/scripts/mvc/workflow/workflow-view.js
+1 -1
View File
@@ -296,7 +296,7 @@ export default Backbone.View.extend({
Save: save_current_workflow,
"Save As": workflow_save_as,
Run: function() {
window.location = `${Galaxy.root}workflow/run?id=${self.options.id}`;
window.location = `${Galaxy.root}workflows/run?id=${self.options.id}`;
},
"Edit Attributes": function() {
self.workflow.clear_active_node();
client/galaxy/scripts/mvc/workflow/workflow.js
+5 -5
View File
@@ -36,7 +36,7 @@ var WorkflowItemView = Backbone.View.extend({
this.model.save();
// This reloads the whole page, so that the workflow appears in the tool panel.
// Ideally we would notify only the tool panel of a change
window.location = `${Galaxy.root}workflow`;
window.location = `${Galaxy.root}workflows/list`;
},
removeWorkflow: function() {
@@ -127,7 +127,7 @@ var WorkflowItemView = Backbone.View.extend({
if (this.model.get("owner") === Galaxy.user.attributes.username) {
return `<ul class="dropdown-menu action-dpd"><li><a href="${Galaxy.root}workflow/editor?id=${
this.model.id
}">Edit</a></li><li><a href="${Galaxy.root}workflow/run?id=${this.model.id}">Run</a></li><li><a href="${
}">Edit</a></li><li><a href="${Galaxy.root}workflows/run?id=${this.model.id}">Run</a></li><li><a href="${
Galaxy.root
}workflow/sharing?id=${this.model.id}">Share</a></li><li><a href="${Galaxy.root}api/workflows/${
this.model.id
@@ -141,7 +141,7 @@ var WorkflowItemView = Backbone.View.extend({
Galaxy.root
}workflow/display_by_username_and_slug?username=${this.model.get("owner")}&slug=${this.model.get(
"slug"
)}">View</a></li><li><a href="${Galaxy.root}workflow/run?id=${
)}">View</a></li><li><a href="${Galaxy.root}workflows/run?id=${
this.model.id
}">Run</a></li><li><a id="copy-workflow" style="cursor: pointer;">Copy</a></li><li><a class="link-confirm-shared-${
this.model.id
@@ -305,9 +305,9 @@ var WorkflowListView = Backbone.View.extend({
_templateActionButtons: function() {
return `<ul class="manage-table-actions"><li><input class="search-wf form-control" type="text" autocomplete="off" placeholder="search for workflow..."></li><li><a class="action-button fa fa-plus wf-action" id="new-workflow" title="Create new workflow" href="${
Galaxy.root
}workflow/create"></a></li><li><a class="action-button fa fa-upload wf-action" id="import-workflow" title="Upload or import workflow" href="${
}workflows/create"></a></li><li><a class="action-button fa fa-upload wf-action" id="import-workflow" title="Upload or import workflow" href="${
Galaxy.root
}workflow/import_workflow"></a></li></ul>`;
}workflows/import_workflow"></a></li></ul>`;
},
/** Template for workflow table */
client/galaxy/scripts/qunit/test.js
+3
View File
@@ -0,0 +1,3 @@
// Load all qunit tests into a single bundle.
var testsContext = require.context(".", true, /_tests$/);
testsContext.keys().forEach(testsContext);
client/galaxy/style/less/ui.less
+3 -3
View File
@@ -507,7 +507,7 @@
position: relative;
clear: both;
width: auto;
height: 100%;
height: auto;
.portlet-header {
background: @form-heading-bg;
border-bottom: solid @form-border 1px;
@@ -550,8 +550,8 @@
width : 100%;
}
.portlet-buttons {
margin-top: @ui-margin-vertical;
margin-bottom: @ui-margin-vertical;
margin-top: @ui-margin-vertical-large;
margin-bottom: @ui-margin-vertical-large;
> .btn {
margin-right: 5px;
}
client/karma.config.js
+44 -35
View File
@@ -1,48 +1,58 @@
var webpackConfig = require('./webpack.config');
// CommonsChunkPlugin not compatible with karma.
// https://github.com/webpack-contrib/karma-webpack/issues/24
webpackConfig.plugins.splice(0, 1);
// Don't build Galaxy bundles - we build per-test bundles.
webpackConfig.entry = function() { return {}; };
// Single pack mode runs whole test suite much more quickly - but requires
// running the whole test suite so it would be slower for one-off tests.
var single_pack_mode = function(){
return process.env.GALAXY_TEST_AS_SINGLE_PACK || false;
};
var TESTS_SEPARATE_PACKS = [
{pattern: 'galaxy/scripts/qunit/tests/list-of-pairs-collection-creator.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/galaxy-app-base.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/graph.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/hda-base.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/history_contents_model_tests.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/job-dag.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/metrics-logger.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/popover_tests.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/utils_test.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/page_tests.js', watched: false},
{pattern: 'galaxy/scripts/qunit/tests/workflow_editor_tests.js', watched: false},
// The following tests don't work for state reasons:
// Error: Following test works on its own or with rest but not with
// list-of-pairs-collection-creator in the same suite. Not as much isolation
// as seperate page setup of previous runner.
// 'galaxy/scripts/qunit/tests/ui_tests.js',
// Error: things displayed wrong I guess cause assertions fail on CSS stuff
// 'galaxy/scripts/qunit/tests/modal_tests.js',
// 'galaxy/scripts/qunit/tests/upload_dialog_tests.js',
// Error: Cannot find module "libs/bibtexParse"
// 'galaxy/scripts/qunit/tests/form_tests.js',
// 'galaxy/scripts/qunit/tests/masthead_tests.js',
];
var TESTS_AS_SINGLE_PACK = [
{pattern: 'galaxy/scripts/qunit/test.js', watched: false},
];
// karma.conf.js
module.exports = function(config) {
config.set({
basepath: '.',
files: [
// Broken into uncommented working tests and commented out broken tests,
// this can be replaced with a glob when they all just work.
// Working tests:
'galaxy/scripts/qunit/tests/list-of-pairs-collection-creator.js',
'galaxy/scripts/qunit/tests/galaxy-app-base.js',
'galaxy/scripts/qunit/tests/graph.js',
'galaxy/scripts/qunit/tests/hda-base.js',
'galaxy/scripts/qunit/tests/history_contents_model_tests.js',
'galaxy/scripts/qunit/tests/job-dag.js',
'galaxy/scripts/qunit/tests/metrics-logger.js',
'galaxy/scripts/qunit/tests/popover_tests.js',
'galaxy/scripts/qunit/tests/utils_test.js',
'galaxy/scripts/qunit/tests/page_tests.js',
'galaxy/scripts/qunit/tests/workflow_editor_tests.js',
// The following tests don't work for state reasons:
// Error: Following test works on its own or with rest but not with
// list-of-pairs-collection-creator in the same suite. Not as much isolation
// as seperate page setup of previous runner.
// 'galaxy/scripts/qunit/tests/ui_tests.js',
// Error: things displayed wrong I guess cause assertions fail on CSS stuff
// 'galaxy/scripts/qunit/tests/modal_tests.js',
// 'galaxy/scripts/qunit/tests/upload_dialog_tests.js',
// Error: Cannot find module "libs/bibtexParse"
// 'galaxy/scripts/qunit/tests/form_tests.js',
// 'galaxy/scripts/qunit/tests/masthead_tests.js',
files: (single_pack_mode() ? TESTS_AS_SINGLE_PACK : TESTS_SEPARATE_PACKS).concat([
// Non-test assets that will be served by web server.
// CSS needed by tests.
'galaxy/scripts/qunit/assets/*.css',
],
]),
plugins: [
'karma-webpack',
'karma-qunit',
@@ -50,7 +60,7 @@ module.exports = function(config) {
'karma-phantomjs-launcher',
],
polyfill: [ 'Object.assign' ],
browsers: [ 'PhantomJS' ],
// browsers: [ 'PhantomJS' ],
// logLevel: config.LOG_DEBUG,
// Tried to do some awesome babel, ES6 stuff in here but I was encountering
@@ -58,13 +68,12 @@ module.exports = function(config) {
// that we have a working setup.
preprocessors: {
// add webpack as preprocessor
'galaxy/scripts/qunit/tests/*.js': ['webpack'],
'galaxy/scripts/qunit/**/*.js': ['webpack']
},
frameworks: ['polyfill', 'qunit'],
webpack: webpackConfig,
webpackMiddleware: { noInfo: false }
});
};
client/package.json
+2 -1
View File
@@ -39,7 +39,8 @@
"webpack-production-maps": "GXY_BUILD_SOURCEMAPS=1 webpack -p",
"style": "grunt style",
"style-watch": "grunt watch-style",
"test": "karma start --verbose --single-run --no-auto-watch karma.config.js",
"test": "karma start --verbose --single-run --browsers PhantomJS karma.config.js",
"test-watch": "karma start --verbose --browsers PhantomJS karma.config.js",
"gulp": "gulp",
"gulp-production": "NODE_ENV=production gulp",
"gulp-production-maps": "GXY_BUILD_SOURCEMAPS=1 NODE_ENV=production gulp",
config/datatypes_conf.xml.sample
+17
View File
@@ -643,6 +643,19 @@
<datatype extension="pttgf" type="galaxy.datatypes.plant_tribes:PlantTribesTargetedGeneFamilies" />
<datatype extension="pttree" type="galaxy.datatypes.plant_tribes:PlantTribesPhylogeneticTree" />
<datatype extension="smat" type="galaxy.datatypes.plant_tribes:Smat" display_in_upload="true" />
<!-- Start Haplotype / LOD Datatypes -->
<datatype extension="alohomora_gts" type="galaxy.datatypes.genetics:GenotypeMatrix" />
<datatype extension="alohomora_map" type="galaxy.datatypes.tabular:Tabular" subclass="true" />
<datatype extension="alohomora_maf" type="galaxy.datatypes.tabular:Tabular" subclass="true" />
<datatype extension="alohomora_ped" type="galaxy.datatypes.tabular:Tabular" subclass="true" />
<!-- Common input formats: Generated by alohomora, but user may also upload these manually -->
<datatype extension="linkage_pedin" type="galaxy.datatypes.tabular:Tabular" subclass="true" />
<datatype extension="linkage_datain" type="galaxy.datatypes.genetics:DataIn" />
<datatype extension="linkage_map" type="galaxy.datatypes.genetics:MarkerMap" />
<!-- All output linkage is converted into the Allegro output format -->
<datatype extension="allegro_ihaplo" type="galaxy.datatypes.tabular:Tabular" />
<datatype extension="allegro_descent" type="galaxy.datatypes.tabular:Tabular" />
<datatype extension="allegro_fparam" type="galaxy.datatypes.genetics:AllegroLOD" />
</registration>
<sniffers>
<!--
@@ -756,6 +769,10 @@
<sniffer type="galaxy.datatypes.text:Ipynb"/>
<sniffer type="galaxy.datatypes.text:Biom1"/>
<sniffer type="galaxy.datatypes.text:Json"/>
<sniffer type="galaxy.datatypes.genetics:GenotypeMatrix" />
<sniffer type="galaxy.datatypes.genetics:DataIn" />
<sniffer type="galaxy.datatypes.genetics:MarkerMap" />
<sniffer type="galaxy.datatypes.genetics:AllegroLOD" />
<sniffer type="galaxy.datatypes.sequence:RNADotPlotMatrix"/>
<sniffer type="galaxy.datatypes.sequence:DotBracket"/>
<sniffer type="galaxy.datatypes.tabular:ConnectivityTable"/>
config/error_report.yml.sample
+2 -1
View File
@@ -29,7 +29,8 @@
# directory: /tmp/reports/
# Submit error reports to sentry. If a sentry_dsn is configured in your
# galaxy.ini, then Galaxy will submit the job error to Sentry.
# galaxy.ini, then Galaxy will submit the job error to Sentry. You may supply a
# separate DSN for tool reports by supplying a ``custom_dsn`` parameter.
- type: sentry
user_submission: false
config/workflow_schedulers_conf.xml.sample
+11
View File
@@ -11,4 +11,15 @@
require any external dependencies. -->
<core id="core" />
<!-- Handlers (Galaxy server processes that perform the scheduling work) can
be defined here in the same format as in job_conf.xml. By default, the
handlers defined in job_conf.xml will be used (or `main` if there is no
job_conf.xml). -->
<!--
<handlers default="handlers">
<handler id="handler0" tags="handlers"/>
<handler id="handler1" tags="handlers"/>
</handlers>
-->
</workflow_schedulers>
lib/galaxy/app.py
+18 -6
View File
@@ -55,12 +55,14 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin):
self.name = 'galaxy'
self.startup_timer = ExecutionTimer()
self.new_installation = False
self.application_stack = application_stack_instance()
# Read config file and check for errors
self.config = config.Configuration(**kwargs)
self.config.check()
config.configure_logging(self.config)
self.configure_fluent_log()
# A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
self.application_stack = application_stack_instance(app=self)
self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
self.config.reload_sanitize_whitelist(explicit='sanitize_whitelist_file' in kwargs)
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
# control_worker *can* be initialized with a queue, but here we don't
@@ -188,10 +190,7 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin):
# Start the job manager
from galaxy.jobs import manager
self.job_manager = manager.JobManager(self)
self.job_manager.start()
# FIXME: These are exposed directly for backward compatibility
self.job_queue = self.job_manager.job_queue
self.job_stop_queue = self.job_manager.job_stop_queue
self.application_stack.register_postfork_function(self.job_manager.start)
self.proxy_manager = ProxyManager(self.config)
# Initialize the external service types
self.external_service_types = external_service_types.ExternalServiceTypesCollection(
@@ -208,11 +207,15 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin):
handlers[signal.SIGUSR1] = self.heartbeat.dump_signal_handler
self._configure_signal_handlers(handlers)
# Start web stack message handling
self.application_stack.register_postfork_function(self.application_stack.start)
self.model.engine.dispose()
self.server_starttime = int(time.time()) # used for cachebusting
log.info("Galaxy app startup finished %s" % self.startup_timer)
def shutdown(self):
log.debug('Shutting down')
exception = None
try:
self.watchers.shutdown()
@@ -258,8 +261,16 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin):
exception = exception or e
log.exception("Failed to shutdown SA database engine cleanly")
try:
self.application_stack.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown application stack interface cleanly")
if exception:
raise exception
else:
log.debug('Finished shutting down')
def configure_fluent_log(self):
if self.config.fluent_log:
@@ -268,5 +279,6 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin):
else:
self.trace_logger = None
@property
def is_job_handler(self):
return (self.config.track_jobs_in_database and self.job_config.is_handler(self.config.server_name)) or not self.config.track_jobs_in_database
return (self.config.track_jobs_in_database and self.job_config.is_handler) or not self.config.track_jobs_in_database
lib/galaxy/config.py
+62 -83
View File
@@ -30,7 +30,7 @@ from galaxy.util import unicodify
from galaxy.util.dbkeys import GenomeBuilds
from galaxy.util.logging import LOGLV_TRACE
from galaxy.web.formatting import expand_pretty_datetime_format
from galaxy.web.stack import register_postfork_function
from galaxy.web.stack import get_stack_facts, register_postfork_function
from .version import VERSION_MAJOR
log = logging.getLogger(__name__)
@@ -79,6 +79,49 @@ PATH_LIST_DEFAULTS = dict(
'config/tool_conf.xml.sample,config/shed_tool_conf.xml']
)
LOGGING_CONFIG_DEFAULT = {
'version': 1,
'root': {
'handlers': ['console'],
'level': 'INFO',
},
'loggers': {
'galaxy': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': 0,
'qualname': 'galaxy',
},
'paste.httpserver.ThreadPool': {
'level': 'WARN',
'qualname': 'paste.httpserver.ThreadPool',
},
'routes.middleware': {
'level': 'WARN',
'qualname': 'routes.middleware',
},
},
'filters': {
'stack': {
'()': 'galaxy.web.stack.application_stack_log_filter',
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'stack',
'level': 'DEBUG',
'stream': 'ext://sys.stderr',
'filters': ['stack'],
},
},
'formatters': {
'stack': {
'()': 'galaxy.web.stack.application_stack_log_formatter',
},
},
}
def resolve_path(path, root):
"""If 'path' is relative make absolute by prepending 'root'"""
@@ -245,8 +288,6 @@ class Configuration(object):
self.collect_outputs_from = [x.strip() for x in kwargs.get('collect_outputs_from', 'new_file_path,job_working_directory').lower().split(',')]
self.template_path = resolve_path(kwargs.get("template_path", "templates"), self.root)
self.template_cache = resolve_path(kwargs.get("template_cache_path", "database/compiled_templates"), self.root)
self.local_job_queue_workers = int(kwargs.get("local_job_queue_workers", "5"))
self.cluster_job_queue_workers = int(kwargs.get("cluster_job_queue_workers", "3"))
self.job_queue_cleanup_interval = int(kwargs.get("job_queue_cleanup_interval", "5"))
self.cluster_files_directory = os.path.abspath(kwargs.get("cluster_files_directory", "database/pbs"))
@@ -269,11 +310,6 @@ class Configuration(object):
self.output_size_limit = int(kwargs.get('output_size_limit', 0))
self.retry_job_output_collection = int(kwargs.get('retry_job_output_collection', 0))
self.check_job_script_integrity = string_as_bool(kwargs.get("check_job_script_integrity", True))
self.job_walltime = kwargs.get('job_walltime', None)
self.job_walltime_delta = None
if self.job_walltime is not None:
h, m, s = [int(v) for v in self.job_walltime.split(':')]
self.job_walltime_delta = timedelta(0, s, 0, 0, m, h)
self.admin_users = kwargs.get("admin_users", "")
self.admin_users_list = [u.strip() for u in self.admin_users.split(',') if u]
self.mailing_join_addr = kwargs.get('mailing_join_addr', 'galaxy-announce-join@bx.psu.edu')
@@ -306,7 +342,6 @@ class Configuration(object):
self.smtp_password = kwargs.get('smtp_password', None)
self.smtp_ssl = kwargs.get('smtp_ssl', None)
self.track_jobs_in_database = string_as_bool(kwargs.get('track_jobs_in_database', 'True'))
self.start_job_runners = listify(kwargs.get('start_job_runners', ''))
self.expose_dataset_path = string_as_bool(kwargs.get('expose_dataset_path', 'False'))
self.expose_potentially_sensitive_job_metrics = string_as_bool(kwargs.get('expose_potentially_sensitive_job_metrics', 'False'))
self.enable_communication_server = string_as_bool(kwargs.get('enable_communication_server', 'False'))
@@ -354,12 +389,7 @@ class Configuration(object):
self.parallelize_workflow_scheduling_within_histories = string_as_bool(kwargs.get('parallelize_workflow_scheduling_within_histories', 'False'))
self.maximum_workflow_invocation_duration = int(kwargs.get("maximum_workflow_invocation_duration", 2678400))
# Per-user Job concurrency limitations
self.cache_user_job_count = string_as_bool(kwargs.get('cache_user_job_count', False))
self.user_job_limit = int(kwargs.get('user_job_limit', 0))
self.registered_user_job_limit = int(kwargs.get('registered_user_job_limit', self.user_job_limit))
self.anonymous_user_job_limit = int(kwargs.get('anonymous_user_job_limit', self.user_job_limit))
self.default_cluster_job_runner = kwargs.get('default_cluster_job_runner', 'local:///')
self.pbs_application_server = kwargs.get('pbs_application_server', "")
self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "")
self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "")
@@ -517,10 +547,12 @@ class Configuration(object):
# Crummy, but PasteScript does not give you a way to determine this
if arg.lower().startswith('--server-name='):
self.server_name = arg.split('=', 1)[-1]
# Allow explicit override of server name in confg params
# Allow explicit override of server name in config params
if "server_name" in kwargs:
self.server_name = kwargs.get("server_name")
# Store all configured server names
# The application stack code may manipulate the server name
self.base_server_name = self.server_name
# Store all configured server names for the message queue routing
self.server_names = []
for section in global_conf_parser.sections():
if section.startswith('server:'):
@@ -549,12 +581,8 @@ class Configuration(object):
self.galaxy_infrastructure_url_set = galaxy_infrastructure_url_set
# Store advanced job management config
self.job_manager = kwargs.get('job_manager', self.server_name).strip()
self.job_handlers = [x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',')]
self.default_job_handlers = [x.strip() for x in kwargs.get('default_job_handlers', ','.join(self.job_handlers)).split(',')]
# Store per-tool runner configs
self.tool_handlers = self.__read_tool_job_config(global_conf_parser, 'galaxy:tool_handlers', 'name')
self.tool_runners = self.__read_tool_job_config(global_conf_parser, 'galaxy:tool_runners', 'url')
# Galaxy messaging (AMQP) configuration options
self.amqp = {}
try:
@@ -590,6 +618,8 @@ class Configuration(object):
self.api_folders = string_as_bool(kwargs.get('api_folders', False))
# This is for testing new library browsing capabilities.
self.new_lib_browse = string_as_bool(kwargs.get('new_lib_browse', False))
# Logging configuration with logging.config.configDict:
self.logging = kwargs.get('logging', None)
# Error logging with sentry
self.sentry_dsn = kwargs.get('sentry_dsn', None)
# Statistics and profiling with statsd
@@ -689,41 +719,6 @@ class Configuration(object):
self.datatypes_config = self.datatypes_config_file
self.tool_configs = self.tool_config_file
def __read_tool_job_config(self, global_conf_parser, section, key):
try:
tool_runners_config = global_conf_parser.items(section)
# Process config to group multiple configs for the same tool.
rval = {}
for entry in tool_runners_config:
tool_config, val = entry
tool = None
runner_dict = {}
if tool_config.find("[") != -1:
# Found tool with additional params; put params in dict.
tool, params = tool_config[:-1].split("[")
param_dict = {}
for param in params.split(","):
name, value = param.split("@")
param_dict[name] = value
runner_dict['params'] = param_dict
else:
tool = tool_config
# Add runner URL.
runner_dict[key] = val
# Create tool entry if necessary.
if tool not in rval:
rval[tool] = []
# Add entry to runners.
rval[tool].append(runner_dict)
return rval
except configparser.NoSectionError:
return {}
def get(self, key, default):
return self.config_dict.get(key, default)
@@ -891,34 +886,18 @@ def configure_logging(config):
paste_configures_logging = False
auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True"))
if auto_configure_logging:
format = config.get("log_format", "%(name)s %(levelname)s %(asctime)s %(message)s")
level = logging._levelNames[config.get("log_level", "DEBUG")]
destination = config.get("log_destination", "stdout")
log.info("Logging at '%s' level to '%s'" % (level, destination))
# Set level
root.setLevel(level)
disable_chatty_loggers = string_as_bool(config.get("auto_configure_logging_disable_chatty", "True"))
if disable_chatty_loggers:
# Turn down paste httpserver logging
if level <= logging.DEBUG:
for chatty_logger in ["paste.httpserver.ThreadPool", "routes.middleware"]:
logging.getLogger(chatty_logger).setLevel(logging.WARN)
# Remove old handlers
for h in root.handlers[:]:
root.removeHandler(h)
# Create handler
if destination == "stdout":
handler = logging.StreamHandler(sys.stdout)
else:
handler = logging.FileHandler(destination)
# Create formatter
formatter = logging.Formatter(format)
# Hook everything up
handler.setFormatter(formatter)
root.addHandler(handler)
# If sentry is configured, also log to it
logging_conf = config.get('logging', None)
if logging_conf is None:
# if using the default logging config, honor the log_level setting
logging_conf = LOGGING_CONFIG_DEFAULT
if config.get('log_level', 'DEBUG') != 'DEBUG':
logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG')
# configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option
for name, conf in logging_conf.get('handlers', {}).items():
if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf:
conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config))
logging_conf['handlers'][name] = conf
logging.config.dictConfig(logging_conf)
if getattr(config, "sentry_dsn", None):
from raven.handlers.logging import SentryHandler
sentry_handler = SentryHandler(config.sentry_dsn)
lib/galaxy/datatypes/annotation.py
+1 -1
View File
@@ -14,7 +14,7 @@ class SnapHmm(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "SNAP HMM model"
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/binary.py
+1 -1
View File
@@ -1088,7 +1088,7 @@ class TwoBit(Binary):
dataset.peek = "Binary TwoBit format nucleotide file"
dataset.blurb = nice_size(dataset.get_size())
else:
return super(TwoBit, self).set_peek(dataset, is_multi_byte)
return super(TwoBit, self).set_peek(dataset)
def display_peek(self, dataset):
try:
lib/galaxy/datatypes/blast.py
+1 -1
View File
@@ -53,7 +53,7 @@ class BlastXml(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = 'NCBI Blast XML data'
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/constructive_solid_geometry.py
+2 -2
View File
@@ -102,7 +102,7 @@ class Ply(object):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Faces: %s, Vertices: %s" % (str(dataset.metadata.face), str(dataset.metadata.vertex))
else:
dataset.peek = 'File does not exist'
@@ -429,7 +429,7 @@ class Vtk(object):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = self.get_blurb(dataset)
else:
dataset.peek = 'File does not exist'
lib/galaxy/datatypes/data.py
+16 -23
View File
@@ -195,7 +195,12 @@ class Data(object):
max_optional_metadata_filesize = property(get_max_optional_metadata_filesize, set_max_optional_metadata_filesize)
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
"""
Set the peek and blurb text
:param is_multi_byte: deprecated
:type is_multi_byte: bool
"""
if not dataset.dataset.purged:
dataset.peek = ''
dataset.blurb = 'data'
@@ -838,7 +843,7 @@ class Text(Data):
"""
if not dataset.dataset.purged:
# The file must exist on disk for the get_file_peek() method
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap)
dataset.peek = get_file_peek(dataset.file_name, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap)
if line_count is None:
# See if line_count is stored in the metadata
if dataset.metadata.data_lines:
@@ -1046,7 +1051,10 @@ def get_test_fname(fname):
def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipchars=None, line_wrap=True):
"""
Returns the first LINE_COUNT lines wrapped to WIDTH
Returns the first LINE_COUNT lines wrapped to WIDTH.
:param is_multi_byte: deprecated
:type is_multi_byte: bool
>>> fname = get_test_fname('4.bed')
>>> get_file_peek(fname, LINE_COUNT=1)
@@ -1061,20 +1069,12 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc
skipchars = []
lines = []
count = 0
file_type = None
data_checked = False
with compression_utils.get_fileobj(file_name, "U") as temp:
while count < LINE_COUNT:
line = temp.readline(WIDTH)
if line and not is_multi_byte and not data_checked:
# See if we have a compressed or binary file
for char in line:
if ord(char) > 128:
file_type = 'binary'
break
data_checked = True
if file_type == 'binary':
break
try:
line = temp.readline(WIDTH)
except UnicodeDecodeError:
return "binary file"
if not line_wrap:
if line.endswith('\n'):
line = line[:-1]
@@ -1091,11 +1091,4 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc
if not skip_line:
lines.append(line)
count += 1
if file_type == 'binary':
text = "%s file" % file_type
else:
try:
text = util.unicodify('\n'.join(lines))
except UnicodeDecodeError:
text = "binary/unknown file"
return text
return '\n'.join(lines)
lib/galaxy/datatypes/genetics.py
+277
View File
@@ -20,6 +20,7 @@ from cgi import escape
from six.moves.urllib.parse import quote_plus
from galaxy.datatypes import metadata
from galaxy.datatypes.data import Text
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.tabular import Tabular
from galaxy.datatypes.text import Html
@@ -31,6 +32,7 @@ verbose = False
# https://genome.ucsc.edu/goldenpath/help/hgGenomeHelp.html
VALID_GENOME_GRAPH_MARKERS = re.compile('^(chr.*|RH.*|rs.*|SNP_.*|CN.*|A_.*)')
VALID_GENOTYPES_LINE = re.compile('^([a-zA-Z0-9]+)(\\s([0-9]{2}|[A-Z]{2}|NC|\?\?))+\\s*$')
class GenomeGraphs(Tabular):
@@ -827,6 +829,281 @@ class MAlist(RexpBase):
substitute_name_with_metadata='base_name', is_binary=True)
class LinkageStudies(Text):
"""
superclass for classical linkage analysis suites
"""
test_files = [
'linkstudies.allegro_fparam', 'linkstudies.alohomora_gts',
'linkstudies.linkage_datain', 'linkstudies.linkage_map'
]
def __init__(self, **kwd):
Text.__init__(self, **kwd)
self.max_lines = 10
class GenotypeMatrix(LinkageStudies):
"""
Sample matrix of genotypes
- GTs as columns
"""
file_ext = "alohomora_gts"
def __init__(self, **kwd):
super(GenotypeMatrix, self).__init__(**kwd)
self.num_cols = -1
def header_check(self, fio):
header_elems = fio.readline().split('\t')
if header_elems[0] != "Name":
return False
try:
return all([int(sid) > 0 for sid in header_elems[1:]])
except ValueError:
return False
return True
def sniff(self, filename):
"""
>>> classname = GenotypeMatrix
>>> from galaxy.datatypes.sniff import get_test_fname
>>> extn_true = classname().file_ext
>>> file_true = get_test_fname("linkstudies." + extn_true)
>>> classname().sniff(file_true)
True
>>> false_files = list(LinkageStudies.test_files)
>>> false_files.remove("linkstudies." + extn_true)
>>> result_true = []
>>> for fname in false_files:
... file_false = get_test_fname(fname)
... res = classname().sniff(file_false)
... if res:
... result_true.append(fname)
>>>
>>> result_true
[]
"""
with open(filename, "r") as fio:
if not self.header_check(fio):
return False
for lcount, line in enumerate(fio):
if lcount > self.max_lines:
return True
tokens = line.split('\t')
if self.num_cols == -1:
self.num_cols = len(tokens)
elif self.num_cols != len(tokens):
return False
if not VALID_GENOTYPES_LINE.match(line):
return False
return True
class MarkerMap(LinkageStudies):
"""
Map of genetic markers including physical and genetic distance
Common input format for linkage programs
chrom, genetic pos, markername, physical pos, Nr
"""
file_ext = "linkage_map"
def header_check(self, fio):
headers = fio.readline().split()
if len(headers) == 5 and headers[0] == "#Chr":
return True
return False
def sniff(self, filename):
"""
>>> classname = MarkerMap
>>> from galaxy.datatypes.sniff import get_test_fname
>>> extn_true = classname().file_ext
>>> file_true = get_test_fname("linkstudies." + extn_true)
>>> classname().sniff(file_true)
True
>>> false_files = list(LinkageStudies.test_files)
>>> false_files.remove("linkstudies." + extn_true)
>>> result_true = []
>>> for fname in false_files:
... file_false = get_test_fname(fname)
... res = classname().sniff(file_false)
... if res:
... result_true.append(fname)
>>>
>>> result_true
[]
"""
with open(filename, "r") as fio:
if not self.header_check(fio):
return False
for lcount, line in enumerate(fio):
if lcount > self.max_lines:
return True
try:
chrm, gpos, nam, bpos, row = line.split()
float(gpos)
int(bpos)
try:
int(chrm)
except ValueError:
if not chrm.lower()[0] in ('x', 'y', 'm'):
return False
except ValueError:
return False
return True
class DataIn(LinkageStudies):
"""
Common linkage input file for intermarker distances
and recombination rates
"""
file_ext = "linkage_datain"
def __init__(self, **kwd):
super(DataIn, self).__init__(**kwd)
self.num_markers = None
self.intermarkers = 0
def eof_function(self):
return self.intermarkers > 0
def sniff(self, filename):
"""
>>> classname = DataIn
>>> from galaxy.datatypes.sniff import get_test_fname
>>> extn_true = classname().file_ext
>>> file_true = get_test_fname("linkstudies." + extn_true)
>>> classname().sniff(file_true)
True
>>> false_files = list(LinkageStudies.test_files)
>>> false_files.remove("linkstudies." + extn_true)
>>> result_true = []
>>> for fname in false_files:
... file_false = get_test_fname(fname)
... res = classname().sniff(file_false)
... if res:
... result_true.append(fname)
>>>
>>> result_true
[]
"""
with open(filename, "r") as fio:
for lcount, line in enumerate(fio):
if lcount > self.max_lines:
return self.eof_function()
tokens = line.split()
try:
if lcount == 0:
self.num_markers = int(tokens[0])
map(int, tokens[1:])
elif lcount == 1:
map(float, tokens)
if len(tokens) != 4:
return False
elif lcount == 2:
map(int, tokens)
last_token = int(tokens[-1])
if self.num_markers is None:
return False
if len(tokens) != last_token:
return False
if self.num_markers != last_token:
return False
elif tokens[0] == "3" and tokens[1] == "2":
self.intermarkers += 1
except (ValueError, IndexError):
return False
return self.eof_function()
class AllegroLOD(LinkageStudies):
"""
Allegro output format for LOD scores
"""
file_ext = "allegro_fparam"
def header_check(self, fio):
header = fio.readline().splitlines()[0].split()
if len(header) == 4 and header == [
"family", "location", "LOD", "marker"
]:
return True
return False
def sniff(self, filename):
"""
>>> classname = AllegroLOD
>>> from galaxy.datatypes.sniff import get_test_fname
>>> extn_true = classname().file_ext
>>> file_true = get_test_fname("linkstudies." + extn_true)
>>> classname().sniff(file_true)
True
>>> false_files = list(LinkageStudies.test_files)
>>> false_files.remove("linkstudies." + extn_true)
>>> result_true = []
>>> for fname in false_files:
... file_false = get_test_fname(fname)
... res = classname().sniff(file_false)
... if res:
... result_true.append(fname)
>>>
>>> result_true
[]
"""
with open(filename, "r") as fio:
if not self.header_check(fio):
return False
for lcount, line in enumerate(fio):
if lcount > self.max_lines:
return True
tokens = line.split()
try:
int(tokens[0])
float(tokens[1])
if tokens[2] != "-inf":
float(tokens[2])
except (ValueError, IndexError):
return False
return True
if __name__ == '__main__':
import doctest
doctest.testmod(sys.modules[__name__])
lib/galaxy/datatypes/graph.py
+2 -2
View File
@@ -27,7 +27,7 @@ class Xgmml(xml.GenericXml):
Set the peek and blurb text
"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'XGMML data'
else:
dataset.peek = 'file does not exist'
@@ -73,7 +73,7 @@ class Sif(tabular.Tabular):
Set the peek and blurb text
"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'SIF data'
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/images.py
+2 -9
View File
@@ -7,7 +7,6 @@ import zipfile
from six.moves.urllib.parse import quote_plus
from galaxy.datatypes.binary import Binary
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.text import Html as HtmlFromText
from galaxy.util import nice_size
from galaxy.util.image_util import check_image_type
@@ -160,14 +159,8 @@ class Pdf(Image):
def sniff(self, filename):
"""Determine if the file is in pdf format."""
headers = get_headers(filename, None, 1)
try:
if headers[0][0].startswith("%PDF"):
return True
else:
return False
except IndexError:
return False
with open(filename, 'rb') as fh:
return fh.read(4) == b"%PDF"
Binary.register_sniffable_binary_format("pdf", "pdf", Pdf)
lib/galaxy/datatypes/molecules.py
+8 -12
View File
@@ -61,12 +61,11 @@ class GenericMolFile(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -471,7 +470,7 @@ class PHAR(GenericMolFile):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "pharmacophore"
else:
dataset.peek = 'file does not exist'
@@ -524,7 +523,7 @@ class PDB(GenericMolFile):
if not dataset.dataset.purged:
atom_numbers = count_special_lines("^ATOM", dataset.file_name)
hetatm_numbers = count_special_lines("^HETATM", dataset.file_name)
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "%s atoms and %s HET-atoms" % (atom_numbers, hetatm_numbers)
else:
dataset.peek = 'file does not exist'
@@ -575,7 +574,7 @@ class PDBQT(GenericMolFile):
if not dataset.dataset.purged:
root_numbers = count_special_lines("^ROOT", dataset.file_name)
branch_numbers = count_special_lines("^BRANCH", dataset.file_name)
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "%s roots and %s branches" % (root_numbers, branch_numbers)
else:
dataset.peek = 'file does not exist'
@@ -587,7 +586,7 @@ class grd(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "grids for docking"
else:
dataset.peek = 'file does not exist'
@@ -621,12 +620,11 @@ class InChI(Tabular):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -666,12 +664,11 @@ class SMILES(Tabular):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if dataset.metadata.number_of_molecules == 1:
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -727,12 +724,11 @@ class CML(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
lib/galaxy/datatypes/msa.py
+3 -5
View File
@@ -17,7 +17,7 @@ class Hmmer(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "HMMER Database"
else:
dataset.peek = 'file does not exist'
@@ -104,12 +104,11 @@ class Stockholm_1_0(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_models == 1):
dataset.blurb = "1 alignment"
else:
dataset.blurb = "%s alignments" % dataset.metadata.number_of_models
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
@@ -187,12 +186,11 @@ class MauveXmfa(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_models == 1):
dataset.blurb = "1 alignment"
else:
dataset.blurb = "%s alignments" % dataset.metadata.number_of_models
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
lib/galaxy/datatypes/plant_tribes.py
+13 -13
View File
@@ -23,7 +23,7 @@ class Smat(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "ESTScan scores matrices"
else:
dataset.peek = 'file does not exist'
@@ -125,7 +125,7 @@ class PlantTribesKsComponents(Tabular):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
if (dataset.metadata.number_comp == 1):
dataset.blurb = "1 significant component"
else:
@@ -159,7 +159,7 @@ class PlantTribesOrtho(PlantTribes):
file_ext = "ptortho"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesOrtho, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesOrtho, self).set_peek(dataset)
dataset.blurb = "Proteins orthogroup fasta files: %d items" % dataset.metadata.num_files
@@ -171,7 +171,7 @@ class PlantTribesOrthoCodingSequence(PlantTribes):
file_ext = "ptorthocs"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesOrthoCodingSequence, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesOrthoCodingSequence, self).set_peek(dataset)
dataset.blurb = "Protein and coding sequences orthogroup fasta files: %d items" % dataset.metadata.num_files
@@ -182,7 +182,7 @@ class PlantTribesTargetedGeneFamilies(PlantTribes):
file_ext = "pttgf"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesTargetedGeneFamilies, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesTargetedGeneFamilies, self).set_peek(dataset)
dataset.blurb = "Targeted gene families"
@@ -194,7 +194,7 @@ class PlantTribesPhylogeneticTree(PlantTribes):
file_ext = "pttree"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesPhylogeneticTree, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesPhylogeneticTree, self).set_peek(dataset)
dataset.blurb = "Phylogenetic trees: %d items" % dataset.metadata.num_files
@@ -205,7 +205,7 @@ class PlantTribesPhylip(PlantTribes):
file_ext = "ptphylip"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesPhylip, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesPhylip, self).set_peek(dataset)
dataset.blurb = "Orthogroup phylip multiple sequence alignments: %d items" % dataset.metadata.num_files
@@ -216,7 +216,7 @@ class PlantTribesMultipleSequenceAlignment(PlantTribes):
file_ext = "ptalign"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignment, self).set_peek(dataset)
dataset.blurb = "Proteins orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -227,7 +227,7 @@ class PlantTribesMultipleSequenceAlignmentCodonAlignment(PlantTribes):
file_ext = "ptalignca"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentCodonAlignment, self).set_peek(dataset)
dataset.blurb = "Protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -238,7 +238,7 @@ class PlantTribesMultipleSequenceAlignmentTrimmed(PlantTribes):
file_ext = "ptaligntrimmed"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentTrimmed, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentTrimmed, self).set_peek(dataset)
dataset.blurb = "Trimmed proteins orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -249,7 +249,7 @@ class PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment(PlantTribes):
file_ext = "ptaligntrimmedca"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment, self).set_peek(dataset)
dataset.blurb = "Trimmed protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -260,7 +260,7 @@ class PlantTribesMultipleSequenceAlignmentFiltered(PlantTribes):
file_ext = "ptalignfiltered"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentFiltered, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentFiltered, self).set_peek(dataset)
dataset.blurb = "Filtered proteins orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -271,5 +271,5 @@ class PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment(PlantTribes):
file_ext = "ptalignfilteredca"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment, self).set_peek(dataset)
dataset.blurb = "Filtered protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files
lib/galaxy/datatypes/proteomics.py
+5 -5
View File
@@ -114,7 +114,7 @@ class ProteomicsXml(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = self.blurb
else:
dataset.peek = 'file does not exist'
@@ -221,7 +221,7 @@ class Mgf(Text):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'mgf Mascot Generic Format'
else:
dataset.peek = 'file does not exist'
@@ -249,7 +249,7 @@ class MascotDat(Text):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'mascotdat Mascot Search Results'
else:
dataset.peek = 'file does not exist'
@@ -334,7 +334,7 @@ class SPLibNoIndex(Text):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Spectral Library without index files'
else:
dataset.peek = 'file does not exist'
@@ -374,7 +374,7 @@ class SPLib(Msp):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'splib Spectral Library Format'
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/registry.py
+8 -7
View File
@@ -314,13 +314,14 @@ class Registry(object):
def append_to_sniff_order():
# Just in case any supported data types are not included in the config's sniff_order section.
for ext, datatype in self.datatypes_by_extension.items():
included = False
for atype in self.sniff_order:
if isinstance(atype, datatype.__class__):
included = True
break
if not included:
self.sniff_order.append(datatype)
if hasattr(datatype, 'sniff'):
included = False
for atype in self.sniff_order:
if isinstance(atype, datatype.__class__):
included = True
break
if not included:
self.sniff_order.append(datatype)
append_to_sniff_order()
def _load_build_sites(self, root):
lib/galaxy/datatypes/sequence.py
+3 -3
View File
@@ -61,7 +61,7 @@ class SequenceSplitLocations(data.Text):
try:
parsed_data = json.load(open(dataset.file_name))
# dataset.peek = json.dumps(data, sort_keys=True, indent=4)
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = '%d sections' % len(parsed_data['sections'])
except Exception:
dataset.peek = 'Not FQTOC file'
@@ -112,7 +112,7 @@ class Sequence(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
if dataset.metadata.sequences:
dataset.blurb = "%s sequences" % util.commaify(str(dataset.metadata.sequences))
else:
@@ -861,7 +861,7 @@ class Maf(Alignment):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
# The file must exist on disk for the get_file_peek() method
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
if dataset.metadata.blocks:
dataset.blurb = "%s blocks" % util.commaify(str(dataset.metadata.blocks))
else:
lib/galaxy/datatypes/sniff.py
+23 -36
View File
@@ -20,7 +20,6 @@ from galaxy.datatypes.binary import Binary
from galaxy.util import (
compression_utils,
multi_byte,
unicodify
)
from galaxy.util.checkers import (
check_binary,
@@ -204,17 +203,11 @@ def convert_newlines_sep2tabs(fname, in_place=True, patt="\\s+", tmp_dir=None, t
return (i + 1, temp_name)
def iter_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=None):
def iter_headers(fname, sep, count=60, comment_designator=None):
with compression_utils.get_fileobj(fname) as in_file:
idx = 0
for line in in_file:
line = line.rstrip('\n\r')
if is_multi_byte:
# TODO: fix this - sep is never found in line
line = unicodify(line, 'utf-8')
sep = sep.encode('utf-8')
if comment_designator is not None and comment_designator != '':
comment_designator = comment_designator.encode('utf-8')
if comment_designator is not None and comment_designator != '' and line.startswith(comment_designator):
continue
yield line.split(sep)
@@ -223,22 +216,22 @@ def iter_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=N
break
def get_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=None):
def get_headers(fname, sep, count=60, comment_designator=None):
"""
Returns a list with the first 'count' lines split by 'sep', ignoring lines
starting with 'comment_designator'
>>> fname = get_test_fname('complete.bed')
>>> get_headers(fname,'\\t')
[['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
>>> get_headers(fname,'\\t') == [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
True
>>> fname = get_test_fname('test.gff')
>>> get_headers(fname, '\\t', count=5, comment_designator='#')
[[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']]
>>> get_headers(fname, '\\t', count=5, comment_designator='#') == [[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']]
True
"""
return list(iter_headers(fname=fname, sep=sep, count=count, is_multi_byte=is_multi_byte, comment_designator=comment_designator))
return list(iter_headers(fname=fname, sep=sep, count=count, comment_designator=comment_designator))
def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False):
def is_column_based(fname, sep='\t', skip=0):
"""
Checks whether the file is column based with respect to a separator
(defaults to tab separator).
@@ -266,7 +259,10 @@ def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False):
>>> is_column_based(fname)
True
"""
headers = get_headers(fname, sep, is_multi_byte=is_multi_byte)
try:
headers = get_headers(fname, sep)
except UnicodeDecodeError:
return False
count = 0
if not headers:
return False
@@ -284,7 +280,7 @@ def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False):
return True
def guess_ext(fname, sniff_order, is_multi_byte=False):
def guess_ext(fname, sniff_order):
"""
Returns an extension that can be used in the datatype factory to
generate a data for the 'fname' file
@@ -393,6 +389,9 @@ def guess_ext(fname, sniff_order, is_multi_byte=False):
>>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom')
>>> guess_ext(fname, sniff_order)
'biom2'
>>> fname = get_test_fname('454Score.pdf')
>>> guess_ext(fname, sniff_order)
'pdf'
"""
file_ext = None
for datatype in sniff_order:
@@ -414,28 +413,16 @@ def guess_ext(fname, sniff_order, is_multi_byte=False):
# to tsv but it doesn't have a sniffer - is TSV was sniffed just check
# if it is an okay tabular and use that instead.
if file_ext == 'tsv':
if is_column_based(fname, '\t', 1, is_multi_byte=is_multi_byte):
if is_column_based(fname, '\t', 1):
file_ext = 'tabular'
if file_ext is not None:
return file_ext
headers = get_headers(fname, None)
is_binary = False
if is_multi_byte:
is_binary = False
else:
for hdr in headers:
for char in hdr:
# old behavior had 'char' possibly having length > 1,
# need to determine when/if this occurs
is_binary = util.is_binary(char)
if is_binary:
break
if is_binary:
break
if is_binary:
try:
get_headers(fname, None)
except UnicodeDecodeError:
return 'data' # default binary data type file extension
if is_column_based(fname, '\t', 1, is_multi_byte=is_multi_byte):
if is_column_based(fname, '\t', 1):
return 'tabular' # default tabular data type file extension
return 'txt' # default text data type file extension
@@ -489,14 +476,14 @@ def handle_compressed_file(filename, datatypes_registry, ext='auto'):
return is_valid, ext
def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto', is_multi_byte=False):
def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto'):
is_valid, ext = handle_compressed_file(filename, datatypes_registry, ext=ext)
if not is_valid:
raise InappropriateDatasetContentError('The compressed uploaded file contains inappropriate content.')
if ext in AUTO_DETECT_EXTENSIONS:
ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order, is_multi_byte=is_multi_byte)
ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order)
if check_binary(filename):
if not Binary.is_ext_unsniffable(ext) and not datatypes_registry.get_datatype_by_extension(ext).sniff(filename):
lib/galaxy/datatypes/tabular.py
+2 -2
View File
@@ -51,7 +51,7 @@ class TabularData(data.Text):
raise NotImplementedError
def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None):
super(TabularData, self).set_peek(dataset, line_count=line_count, is_multi_byte=is_multi_byte, WIDTH=WIDTH, skipchars=skipchars, line_wrap=False)
super(TabularData, self).set_peek(dataset, line_count=line_count, WIDTH=WIDTH, skipchars=skipchars, line_wrap=False)
if dataset.metadata.comment_lines:
dataset.blurb = "%s, %s comments" % (dataset.blurb, util.commaify(str(dataset.metadata.comment_lines)))
@@ -825,7 +825,7 @@ class Eland(Tabular):
- LANE, TILEm X, Y, INDEX, READ_NO, SEQ, QUAL, POSITION, *STRAND, FILT must be correct
- We will only check that up to the first 5 alignments are correctly formatted.
"""
with compression_utils.get_fileobj(filename, gzip_only=True) as fh:
with compression_utils.get_fileobj(filename, compressed_formats=['gzip']) as fh:
count = 0
while True:
line = fh.readline()
lib/galaxy/datatypes/test/454Score.pdf
+545
View File
@@ -0,0 +1,545 @@
%PDF-1.1
%�â�ã�Ï�Ó\r
1 0 obj
<<
/CreationDate (D:20080403110358)
/ModDate (D:20080403110358)
/Title (R Graphics Output)
/Producer (R 2.6.2)
/Creator (R)
>>
endobj
2 0 obj
<<
/Type /Catalog
/Pages 3 0 R
>>
endobj
5 0 obj
<<
/Type /Font
/Subtype /Type1
/Name /F1
/BaseFont /ZapfDingbats
>>
endobj
6 0 obj
<<
/Type /Page
/Parent 3 0 R
/Contents 7 0 R
/Resources 4 0 R
>>
endobj
7 0 obj
<<
/Length 8 0 R
>>
stream
q
Q q 59.04 73.44 342.72 299.52 re W n
0.000 0.000 0.000 RG
2.25 w
[] 0 d
1 J
1 j
10.00 M
73.40 149.79 m 86.76 149.79 l S
0.75 w
[ 3.00 5.00] 0 d
80.08 100.85 m 80.08 149.79 l S
80.08 296.61 m 80.08 263.98 l S
0.75 w
[] 0 d
76.74 100.85 m 83.42 100.85 l S
76.74 296.61 m 83.42 296.61 l S
73.40 149.79 m
86.76 149.79 l
86.76 263.98 l
73.40 263.98 l
73.40 149.79 l
S
2.25 w
[] 0 d
90.11 280.30 m 103.47 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
96.79 263.98 m 96.79 280.30 l S
96.79 296.61 m 96.79 296.61 l S
0.75 w
[] 0 d
93.45 263.98 m 100.13 263.98 l S
93.45 296.61 m 100.13 296.61 l S
90.11 280.30 m
103.47 280.30 l
103.47 296.61 l
90.11 296.61 l
90.11 280.30 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 93.82 342.96 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
106.81 280.30 m 120.17 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
113.49 263.98 m 113.49 263.98 l S
113.49 280.30 m 113.49 280.30 l S
0.75 w
[] 0 d
110.15 263.98 m 116.83 263.98 l S
110.15 280.30 m 116.83 280.30 l S
106.81 263.98 m
120.17 263.98 l
120.17 280.30 l
106.81 280.30 l
106.81 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 110.53 179.82 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
123.51 247.67 m 136.87 247.67 l S
0.75 w
[ 3.00 5.00] 0 d
130.19 247.67 m 130.19 247.67 l S
130.19 247.67 m 130.19 247.67 l S
0.75 w
[] 0 d
126.85 247.67 m 133.53 247.67 l S
126.85 247.67 m 133.53 247.67 l S
123.51 247.67 m
136.87 247.67 l
136.87 247.67 l
123.51 247.67 l
123.51 247.67 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 127.23 261.39 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
140.21 280.30 m 153.57 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
146.89 231.36 m 146.89 247.67 l S
146.89 361.87 m 146.89 296.61 l S
0.75 w
[] 0 d
143.55 231.36 m 150.23 231.36 l S
143.55 361.87 m 150.23 361.87 l S
140.21 247.67 m
153.57 247.67 l
153.57 296.61 l
140.21 296.61 l
140.21 247.67 l
S
2.25 w
[] 0 d
156.91 280.30 m 170.27 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
163.59 198.73 m 163.59 231.36 l S
163.59 280.30 m 163.59 280.30 l S
0.75 w
[] 0 d
160.25 198.73 m 166.93 198.73 l S
160.25 280.30 m 166.93 280.30 l S
156.91 231.36 m
170.27 231.36 l
170.27 280.30 l
156.91 280.30 l
156.91 231.36 l
S
2.25 w
[] 0 d
173.61 280.30 m 186.98 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
180.29 247.67 m 180.29 263.98 l S
180.29 280.30 m 180.29 280.30 l S
0.75 w
[] 0 d
176.95 247.67 m 183.64 247.67 l S
176.95 280.30 m 183.64 280.30 l S
173.61 263.98 m
186.98 263.98 l
186.98 280.30 l
173.61 280.30 l
173.61 263.98 l
S
2.25 w
[] 0 d
190.32 247.67 m 203.68 247.67 l S
0.75 w
[ 3.00 5.00] 0 d
197.00 247.67 m 197.00 247.67 l S
197.00 263.98 m 197.00 263.98 l S
0.75 w
[] 0 d
193.66 247.67 m 200.34 247.67 l S
193.66 263.98 m 200.34 263.98 l S
190.32 247.67 m
203.68 247.67 l
203.68 263.98 l
190.32 263.98 l
190.32 247.67 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 194.03 294.02 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
207.02 263.98 m 220.38 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
213.70 247.67 m 213.70 247.67 l S
213.70 263.98 m 213.70 263.98 l S
0.75 w
[] 0 d
210.36 247.67 m 217.04 247.67 l S
210.36 263.98 m 217.04 263.98 l S
207.02 247.67 m
220.38 247.67 l
220.38 263.98 l
207.02 263.98 l
207.02 247.67 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 210.74 342.96 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
223.72 263.98 m 237.08 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
230.40 263.98 m 230.40 263.98 l S
230.40 280.30 m 230.40 280.30 l S
0.75 w
[] 0 d
227.06 263.98 m 233.74 263.98 l S
227.06 280.30 m 233.74 280.30 l S
223.72 263.98 m
237.08 263.98 l
237.08 280.30 l
223.72 280.30 l
223.72 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 227.44 163.51 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
240.42 280.30 m 253.78 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
247.10 263.98 m 247.10 263.98 l S
247.10 280.30 m 247.10 280.30 l S
0.75 w
[] 0 d
243.76 263.98 m 250.44 263.98 l S
243.76 280.30 m 250.44 280.30 l S
240.42 263.98 m
253.78 263.98 l
253.78 280.30 l
240.42 280.30 l
240.42 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 244.14 179.82 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
257.12 263.98 m 270.48 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
263.80 247.67 m 263.80 247.67 l S
263.80 280.30 m 263.80 280.30 l S
0.75 w
[] 0 d
260.46 247.67 m 267.14 247.67 l S
260.46 280.30 m 267.14 280.30 l S
257.12 247.67 m
270.48 247.67 l
270.48 280.30 l
257.12 280.30 l
257.12 247.67 l
S
2.25 w
[] 0 d
273.82 247.67 m 287.19 247.67 l S
0.75 w
[ 3.00 5.00] 0 d
280.51 84.53 m 280.51 182.42 l S
280.51 296.61 m 280.51 296.61 l S
0.75 w
[] 0 d
277.16 84.53 m 283.85 84.53 l S
277.16 296.61 m 283.85 296.61 l S
273.82 182.42 m
287.19 182.42 l
287.19 296.61 l
273.82 296.61 l
273.82 182.42 l
S
2.25 w
[] 0 d
290.53 280.30 m 303.89 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
297.21 280.30 m 297.21 280.30 l S
297.21 280.30 m 297.21 280.30 l S
0.75 w
[] 0 d
293.87 280.30 m 300.55 280.30 l S
293.87 280.30 m 300.55 280.30 l S
290.53 280.30 m
303.89 280.30 l
303.89 280.30 l
290.53 280.30 l
290.53 280.30 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 294.25 294.02 Tm (l) Tj 0 Tr
/F1 1 Tf 1 Tr 7.48 0 0 7.48 294.25 228.76 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
307.23 263.98 m 320.59 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
313.91 247.67 m 313.91 247.67 l S
313.91 280.30 m 313.91 280.30 l S
0.75 w
[] 0 d
310.57 247.67 m 317.25 247.67 l S
310.57 280.30 m 317.25 280.30 l S
307.23 247.67 m
320.59 247.67 l
320.59 280.30 l
307.23 280.30 l
307.23 247.67 l
S
2.25 w
[] 0 d
323.93 231.36 m 337.29 231.36 l S
0.75 w
[ 3.00 5.00] 0 d
330.61 198.73 m 330.61 215.04 l S
330.61 231.36 m 330.61 231.36 l S
0.75 w
[] 0 d
327.27 198.73 m 333.95 198.73 l S
327.27 231.36 m 333.95 231.36 l S
323.93 215.04 m
337.29 215.04 l
337.29 231.36 l
323.93 231.36 l
323.93 215.04 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 327.65 261.39 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
340.63 263.98 m 353.99 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
347.31 263.98 m 347.31 263.98 l S
347.31 263.98 m 347.31 263.98 l S
0.75 w
[] 0 d
343.97 263.98 m 350.65 263.98 l S
343.97 263.98 m 350.65 263.98 l S
340.63 263.98 m
353.99 263.98 l
353.99 263.98 l
340.63 263.98 l
340.63 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 344.35 179.82 Tm (l) Tj 0 Tr
/F1 1 Tf 1 Tr 7.48 0 0 7.48 344.35 277.70 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
357.33 263.98 m 370.69 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
364.01 247.67 m 364.01 247.67 l S
364.01 280.30 m 364.01 280.30 l S
0.75 w
[] 0 d
360.67 247.67 m 367.35 247.67 l S
360.67 280.30 m 367.35 280.30 l S
357.33 247.67 m
370.69 247.67 l
370.69 280.30 l
357.33 280.30 l
357.33 247.67 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 361.05 81.94 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
374.04 263.98 m 387.40 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
380.72 263.98 m 380.72 263.98 l S
380.72 296.61 m 380.72 296.61 l S
0.75 w
[] 0 d
377.38 263.98 m 384.06 263.98 l S
377.38 296.61 m 384.06 296.61 l S
374.04 263.98 m
387.40 263.98 l
387.40 296.61 l
374.04 296.61 l
374.04 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 377.75 163.51 Tm (l) Tj 0 Tr
ET
Q q
0.000 0.000 0.000 RG
0.75 w
[] 0 d
1 J
1 j
10.00 M
59.04 133.47 m 59.04 296.61 l S
59.04 133.47 m 51.84 133.47 l S
59.04 215.04 m 51.84 215.04 l S
59.04 296.61 m 51.84 296.61 l S
BT
0.000 0.000 0.000 rg
/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 126.80 Tm (20) Tj
/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 208.37 Tm (25) Tj
/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 289.94 Tm (30) Tj
ET
Q q
BT
0.000 0.000 0.000 rg
/F3 1 Tf 14.00 0.00 -0.00 14.00 147.76 397.45 Tm (boxplot of quality scores) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 130.36 18.72 Tm (position within read \(% of total length\)) Tj
ET
Q q
0.000 0.000 0.000 RG
0.75 w
[] 0 d
1 J
1 j
10.00 M
59.04 73.44 m
401.76 73.44 l
401.76 372.96 l
59.04 372.96 l
59.04 73.44 l
S
63.38 73.44 m 380.72 73.44 l S
63.38 73.44 m 63.38 66.24 l S
80.08 73.44 m 80.08 66.24 l S
96.79 73.44 m 96.79 66.24 l S
113.49 73.44 m 113.49 66.24 l S
130.19 73.44 m 130.19 66.24 l S
146.89 73.44 m 146.89 66.24 l S
163.59 73.44 m 163.59 66.24 l S
180.29 73.44 m 180.29 66.24 l S
197.00 73.44 m 197.00 66.24 l S
213.70 73.44 m 213.70 66.24 l S
230.40 73.44 m 230.40 66.24 l S
247.10 73.44 m 247.10 66.24 l S
263.80 73.44 m 263.80 66.24 l S
280.51 73.44 m 280.51 66.24 l S
297.21 73.44 m 297.21 66.24 l S
313.91 73.44 m 313.91 66.24 l S
330.61 73.44 m 330.61 66.24 l S
347.31 73.44 m 347.31 66.24 l S
364.01 73.44 m 364.01 66.24 l S
380.72 73.44 m 380.72 66.24 l S
BT
0.000 0.000 0.000 rg
/F2 1 Tf 12.00 0.00 -0.00 12.00 60.05 47.52 Tm (0) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 76.75 47.52 Tm (5) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 106.82 47.52 Tm (15) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 140.22 47.52 Tm (25) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 173.62 47.52 Tm (35) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 207.03 47.52 Tm (45) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 240.43 47.52 Tm (55) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 273.83 47.52 Tm (65) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 307.24 47.52 Tm (75) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 340.64 47.52 Tm (85) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 374.04 47.52 Tm (95) Tj
ET
Q
endstream
endobj
8 0 obj
8714
endobj
3 0 obj
<<
/Type /Pages
/Kids [
6 0 R
]
/Count 1
/MediaBox [0 0 432 432]
>>
endobj
4 0 obj
<<
/ProcSet [/PDF /Text]
/Font << /F1 5 0 R /F2 10 0 R /F3 11 0 R >>
/ExtGState << >>
>>
endobj
9 0 obj
<<
/Type /Encoding
/BaseEncoding /WinAnsiEncoding
/Differences [ 45/minus 96/quoteleft
144/dotlessi /grave /acute /circumflex /tilde /macron /breve /dotaccent
/dieresis /.notdef /ring /cedilla /.notdef /hungarumlaut /ogonek /caron /space]
>>
endobj
10 0 obj <<
/Type /Font
/Subtype /Type1
/Name /F2
/BaseFont /Helvetica
/Encoding 9 0 R
>> endobj
11 0 obj <<
/Type /Font
/Subtype /Type1
/Name /F3
/BaseFont /Helvetica-Bold
/Encoding 9 0 R
>> endobj
xref
0 12
0000000000 65535 f
0000000021 00000 n
0000000163 00000 n
0000009162 00000 n
0000009245 00000 n
0000000212 00000 n
0000000295 00000 n
0000000375 00000 n
0000009142 00000 n
0000009349 00000 n
0000009606 00000 n
0000009703 00000 n
trailer
<<
/Size 12
/Info 1 0 R
/Root 2 0 R
>>
startxref
9805
%%EOF
lib/galaxy/datatypes/test/linkstudies.allegro_fparam
+100
View File
@@ -0,0 +1,100 @@
family location LOD marker
1 0.000 -inf rs9617264
1 0.180 -5.9958 rs5746647
1 0.233 -inf rs5747988
1 2.123 -9.4599 rs4819979
1 2.198 -5.2942 rs4266110
1 2.253 -5.0474 rs5748981
1 2.358 -4.9631 rs12483926
1 2.539 -inf rs17807605
1 2.685 -inf rs5749001
1 2.741 -5.4371 rs5749010
1 2.851 -5.1217 rs4819994
1 2.906 -5.1123 rs9617982
1 2.996 -5.2452 rs9619055
1 3.101 -inf rs361626
1 3.232 -5.3301 rs5994287
1 3.293 -inf rs9604737
1 3.451 -8.1586 rs5747087
1 3.507 -24.9936 rs7286310
1 3.557 -inf rs2189077
1 3.609 -8.2376 rs5747183
1 3.861 -11.0094 rs174293
1 3.927 -inf rs1296805
1 4.024 -inf rs5992746
1 4.086 -9.0399 rs3747023
1 4.138 -inf rs5747339
1 4.232 -inf rs5747351
1 4.375 -inf rs443627
1 4.524 -inf rs389496
1 4.586 -inf rs5992854
1 4.649 -inf rs2587115
1 4.702 -inf rs5992126
1 4.753 -inf rs443983
1 4.972 -inf rs390495
1 5.041 -inf rs462904
1 5.187 -22.4923 rs5992985
1 5.279 -inf rs462055
1 5.358 -7.3545 rs362043
1 5.441 -6.9638 rs2540620
1 6.072 -inf rs2543958
1 6.158 -11.2824 rs418623
1 6.219 -6.9834 rs759406
1 6.293 -6.9610 rs5747950
1 6.360 -7.0468 rs5993463
1 6.471 -11.2945 rs737923
1 21.656 -17.9844 rs2187966
1 22.016 -8.9528 rs9624813
1 22.162 -inf rs742030
1 22.221 -inf rs1467387
1 22.334 -8.9315 rs542162
1 22.457 -inf rs12627968
1 22.551 -inf rs133885
1 22.633 -25.6794 rs713816
1 22.688 -inf rs6004774
1 22.756 -inf rs2301492
1 22.911 -inf rs2157538
1 23.108 -8.8047 rs8137311
1 23.211 -inf rs1013815
1 23.322 -inf rs4820663
1 23.395 -inf rs9620577
1 23.624 -7.6981 rs8136102
1 23.760 -inf rs12628683
1 23.908 -inf rs5752264
1 24.053 -inf rs134760
1 24.111 -10.3274 rs2331215
1 24.164 -inf rs667270
1 24.240 -9.4447 rs8138440
1 24.318 -inf rs633132
1 24.393 -7.3251 rs4822702
1 24.515 -7.1150 rs5761487
1 24.608 -inf rs1941126
1 24.856 -7.2865 rs600878
1 24.946 -22.4631 rs134157
1 25.003 -inf rs9613212
1 25.054 -11.5789 rs2071862
1 25.106 -inf rs4822768
1 25.215 -14.0176 rs2001121
1 25.420 -8.4477 rs6005159
1 25.492 -8.6606 rs4820706
1 25.622 -inf rs4822791
1 25.865 -inf rs1476033
1 25.997 -inf rs6005189
1 26.105 -8.1743 rs5761777
1 26.193 -8.0074 rs5761805
1 26.372 -8.2580 rs6005232
1 26.447 -inf rs7290832
1 26.674 -20.6108 rs6005254
1 26.803 -18.5282 rs2516082
1 26.906 -inf rs6005266
1 27.050 -inf rs4418
1 27.198 -inf rs79037
1 27.268 -inf rs5761937
1 27.358 -inf rs5761957
1 27.483 -inf rs134949
1 37.021 -inf rs5749711
1 37.135 -10.6102 rs242990
1 37.204 -10.3303 rs12485116
1 37.277 -inf rs5999265
1 37.363 -inf rs5755038
1 37.474 -inf rs9621951
lib/galaxy/datatypes/test/linkstudies.alohomora_gts
+80
View File
@@ -0,0 +1,80 @@
Name 206007 2060011 206004 2060010 2060016 2060012 2060017 2060013 206008 206001 206002 206
rs3094315 AA AB AB AA AA AB AA AB AA AB AA
rs2073813 BB AB AB BB BB AB BB AB BB AB BB
rs2905040 BB BB BB BB BB BB BB BB BB BB BB
rs12124819 AA AA AA AB AB AA AB AB BB AB AB
rs2980314 BB BB BB BB BB BB BB BB BB BB BB
rs6684487 BB BB BB BB BB BB BB BB BB BB BB
rs4245756 BB BB BB BB BB BB BB BB BB BB BB
rs12086311 BB BB BB BB BB BB BB BB BB BB BB
rs13303077 BB BB BB BB BB BB BB BB BB BB BB
rs28625089 BB BB BB BB BB BB BB BB BB BB BB
rs4475691 BB BB BB BB BB BB BB BB BB BB BB
rs28587382 BB BB BB BB BB BB BB BB BB BB BB
rs13303029 BB BB BB BB BB BB BB BB BB BB BB
rs2340589 BB BB BB BB BB BB BB BB BB BB BB
rs28576697 AA AB BB AA AA AB AA AB AA AB AB
rs1110052 AA AB BB AA AA AB AA AB AA AB AB
rs7523549 BB BB BB AB BB BB BB BB AB AB BB
rs2272756 BB AB AA BB BB AB BB AB BB AB AB
rs28711536 BB BB BB BB BB BB BB BB BB BB BB
rs13303010 AA AA AA AB AA AA AA AA AB AB AA
rs3935066 AA AA AA AB AA AA AA AA AB AB AA
rs13302925 BB BB BB BB BB BB BB BB BB BB BB
rs13303355 BB BB BB BB BB BB BB BB BB BB BB
rs6693747 AB AB AA BB AB AB AB AB AB AB AA
rs9777703 AA AA AA AA AA AA AA AA AA AA AA
rs9697457 BB BB BB BB BB BB BB BB BB BB BB
rs28360890 BB BB BB BB BB BB BB BB BB BB BB
rs3128117 AB AB BB AB AB AB AB AB BB BB BB
rs28446115 BB BB BB BB BB BB BB BB BB BB BB
rs28555183 BB BB BB BB BB BB BB BB BB BB BB
rs4970393 AB AB BB AB AB AB AB AB BB BB BB
rs4970349 AB AB BB AB AB AB AB AB BB BB BB
rs17160776 BB BB BB BB BB BB BB BB BB BB BB
rs2245754 AA AA AB AA AB AA AB AB AB AB AB
rs7526076 AB AB AA AB AB AB AB AB AA AA AA
rs3934834 BB BB BB BB AB BB AB BB BB BB BB
rs3766191 BB BB BB BB AB BB AB BB BB BB BB
rs9442372 AB AB AA AB AA AB AA AB AA AA AA
rs3737728 AB AB AA AB AB AB AB AB AA AA AA
rs10907178 AA AA AA AA AB AA AB AA AA AA AA
rs6687776 BB AB AB BB AB AB AB BB BB BB AB
rs11579015 AA AA AA AA AA AA AA AA AA AA AA
rs6671356 AA AA AA AA AB AA AB AA AA AA AA
rs11811175 BB BB BB BB BB BB BB BB BB BB BB
rs4970405 AA AA AA AA AA AA AA AA AA AA AA
rs1105065 BB BB BB BB AB BB AB BB BB BB BB
rs7540009 BB BB BB BB AB BB AB BB BB BB BB
rs10907182 AB AB AB AB AA AB AA AB AA AB AA
rs12737205 BB BB BB BB AB BB AB BB BB BB BB
rs11579425 AB BB BB AB AB BB AB BB AA AB AB
rs4970416 BB BB BB BB BB BB BB BB BB BB BB
rs4970362 AB BB BB BB BB BB BB AB BB BB BB
rs9660710 BB BB BB BB BB BB BB AB BB BB BB
rs4442317 AA BB AB AA AA BB AA AB AA AA AB
rs12092254 BB BB AB BB AB BB AB AB BB AB BB
rs6668667 BB BB BB BB AB BB AB BB BB BB BB
rs3813204 BB AB AB BB BB AB BB AB BB AB BB
rs4314833 AA AB AB AA AA AB AA AB AA AB AA
rs12060422 AB BB BB AB AB BB AB BB BB BB BB
rs9729550 AB AB AB AB AB AB AB AB AA AB AA
rs11466681 BB BB BB BB BB BB BB BB BB BB BB
rs34945898 BB BB BB BB BB BB BB BB BB BB BB
rs12036216 AB BB BB BB AB BB AB BB BB BB BB
rs3813199 AB BB BB BB AB BB AB BB BB BB BB
rs11260562 BB BB BB BB BB BB BB BB BB BB BB
rs7528416 AB AA AA AA AB AA AB AA AA AA AA
rs715643 BB BB BB BB BB BB BB BB BB BB BB
rs12093154 AB BB BB BB AB BB AB BB BB BB BB
rs6692115 NC NC NC NC NC NC NC NC NC NC NC
rs7524470 AA AA AA AA AA AA AA AA AA AA AA
rs6704013 BB BB BB BB BB BB BB BB BB BB BB
rs4018608 AA AB AB AA AA AB AA AA AA AA AB
rs12073590 AA AA AA AA AA AA AA AA AA AA AA
rs6689813 AA AB AB AA AA AB AA AA AA AA AB
rs34778240 BB BB BB BB BB BB BB BB BB BB BB
rs12077700 BB AB AB BB BB AB BB BB BB BB AB
rs1749951 BB BB BB BB BB BB BB BB BB BB BB
rs11586188 BB BB BB BB BB BB BB BB BB BB BB
rs35573647 BB BB BB BB BB BB BB BB BB BB BB
lib/galaxy/datatypes/test/linkstudies.linkage_datain
+1172
View File
File diff suppressed because one or more lines are too long
lib/galaxy/datatypes/test/linkstudies.linkage_map
+50
View File
@@ -0,0 +1,50 @@
#Chr Genpos Marker Physpos Nr
21 0.70496629 rs1296971 13609442 1
21 0.77494263 rs468601 13769165 2
21 0.85010932 rs2821973 13899316 3
21 1.01291458 rs1929150 14051249 4
21 1.07552959 rs2822124 14088675 5
21 1.16577144 rs2775054 14121682 6
21 1.34457997 rs7276618 14197852 7
21 1.44169062 rs392812 14252347 8
21 1.60661592 rs2822368 14311592 9
21 1.73090413 rs6516610 14385827 10
21 1.78470487 rs447455 14502698 11
21 1.89865013 rs437521 14513560 12
21 1.95928889 rs2822554 14585162 13
21 2.00968909 rs2822572 14618073 14
21 2.09569547 rs13050350 14653413 15
21 2.14611500 rs2142236 14748155 16
21 2.30459845 rs2822677 14767432 17
21 2.35507107 rs2822696 14783326 18
21 2.49458320 rs376635 14788185 19
21 2.54981913 rs2822765 14843387 20
21 2.69205997 rs2822780 14875052 21
21 2.74542953 rs465340 14884525 22
21 2.80926447 rs1888398 14981052 23
21 2.86888387 rs458052 15013563 24
21 2.93942924 rs2822907 15057767 25
21 2.99612354 rs8129531 15061195 26
21 3.15050759 rs12053660 15067326 27
21 3.48398239 rs1883003 15147347 28
21 3.62725156 rs2822974 15185311 29
21 3.69755891 rs11088231 15247441 30
21 3.83233721 rs2205239 15365789 31
21 4.03050303 rs2823045 15382630 32
21 4.11519240 rs926164 15435118 33
21 4.17500340 rs2823139 15498654 34
21 4.22973041 rs2823161 15515055 35
21 4.31848934 rs2823194 15568649 36
21 4.53372425 rs2049882 15633372 37
21 4.62105207 rs1736148 15735083 38
21 4.80636119 rs2823301 15780469 39
21 4.88644465 rs6517467 15806025 40
21 4.96357163 rs2064051 15860950 41
21 5.04209154 rs9974915 15879012 42
21 5.09648993 rs2823400 15922925 43
21 5.28044978 rs726634 15952748 44
21 5.33607360 rs7283707 16048865 45
21 5.52020653 rs7283161 16247081 46
21 5.60568022 rs9982633 16341775 47
21 5.70676016 rs2823621 16453860 48
21 5.81125072 rs2051347 16485140 49
lib/galaxy/datatypes/text.py
+16 -20
View File
@@ -50,13 +50,10 @@ class Html(Text):
True
"""
headers = iter_headers(filename, None)
try:
for i, hdr in enumerate(headers):
if hdr and hdr[0].lower().find('<html>') >= 0:
return True
return False
except Exception:
return True
for i, hdr in enumerate(headers):
if hdr and hdr[0].lower().find('<html>') >= 0:
return True
return False
class Json(Text):
@@ -65,7 +62,7 @@ class Json(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "JavaScript Object Notation (JSON)"
else:
dataset.peek = 'file does not exist'
@@ -113,7 +110,7 @@ class Ipynb(Json):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Jupyter Notebook"
else:
dataset.peek = 'file does not exist'
@@ -186,7 +183,7 @@ class Biom1(Json):
MetadataElement(name="table_columns", default=[], desc="table_columns", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[])
def set_peek(self, dataset, is_multi_byte=False):
super(Biom1, self).set_peek(dataset, is_multi_byte)
super(Biom1, self).set_peek(dataset)
if not dataset.dataset.purged:
dataset.blurb = "Biological Observation Matrix v1"
@@ -270,7 +267,7 @@ class Obo(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Open Biomedical Ontology (OBO)"
else:
dataset.peek = 'file does not exist'
@@ -309,7 +306,7 @@ class Arff(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Attribute-Relation File Format (ARFF)"
dataset.blurb += ", %s comments, %s attributes" % (dataset.metadata.comment_lines, dataset.metadata.columns)
else:
@@ -503,20 +500,19 @@ class SnpSiftDbNSFP(Text):
This is called only at upload to write the html file
cannot rename the datasets here - they come with the default unfortunately
"""
self.regenerate_primary_file(dataset)
return '<html><head><title>SnpSiftDbNSFP Composite Dataset</title></head></html>'
def regenerate_primary_file(self, dataset):
"""
cannot do this until we are setting metadata
"""
annotations = "dbNSFP Annotations: %s\n" % ','.join(dataset.metadata.annotation)
f = open(dataset.file_name, 'a')
if dataset.metadata.bgzip:
bn = dataset.metadata.bgzip
f.write(bn)
f.write('\n')
f.write(annotations)
f.close()
with open(dataset.file_name, 'a') as f:
if dataset.metadata.bgzip:
bn = dataset.metadata.bgzip
f.write(bn)
f.write('\n')
f.write(annotations)
def set_meta(self, dataset, overwrite=True, **kwd):
try:
lib/galaxy/datatypes/triples.py
+7 -7
View File
@@ -31,7 +31,7 @@ class Triples(data.Data):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Triple data'
else:
dataset.peek = 'file does not exist'
@@ -55,7 +55,7 @@ class NTriples(data.Text, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'N-Triples triple data'
else:
dataset.peek = 'file does not exist'
@@ -78,7 +78,7 @@ class N3(data.Text, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Notation-3 Triple data'
else:
dataset.peek = 'file does not exist'
@@ -105,7 +105,7 @@ class Turtle(data.Text, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Turtle triple data'
else:
dataset.peek = 'file does not exist'
@@ -132,7 +132,7 @@ class Rdf(xml.GenericXml, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'RDF/XML triple data'
else:
dataset.peek = 'file does not exist'
@@ -158,7 +158,7 @@ class Jsonld(text.Json, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'JSON-LD triple data'
else:
dataset.peek = 'file does not exist'
@@ -181,7 +181,7 @@ class HDT(binary.Binary, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'HDT triple data'
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/xml.py
+5 -5
View File
@@ -21,7 +21,7 @@ class GenericXml(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'XML data'
else:
dataset.peek = 'file does not exist'
@@ -68,7 +68,7 @@ class MEMEXml(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'MEME XML data'
else:
dataset.peek = 'file does not exist'
@@ -85,7 +85,7 @@ class CisML(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'CisML data'
else:
dataset.peek = 'file does not exist'
@@ -104,7 +104,7 @@ class Phyloxml(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Phyloxml data'
else:
dataset.peek = 'file does not exist'
@@ -139,7 +139,7 @@ class Owl(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = "Web Ontology Language OWL"
else:
dataset.peek = 'file does not exist'
lib/galaxy/jobs/__init__.py
+58 -88
View File
@@ -46,6 +46,7 @@ TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey']
# Override with config.default_job_shell.
DEFAULT_JOB_SHELL = '/bin/bash'
DEFAULT_LOCAL_WORKERS = 4
DEFAULT_CLEANUP_JOB = "always"
@@ -139,7 +140,15 @@ class JobConfiguration(object, ConfiguresHandlers):
self.resource_groups = {}
self.default_resource_group = None
self.resource_parameters = {}
self.limits = Bunch()
self.limits = Bunch(registered_user_concurrent_jobs=None,
anonymous_user_concurrent_jobs=None,
walltime=None,
walltime_delta=None,
total_walltime={},
output_size=None,
destination_user_concurrent_jobs={},
destination_total_concurrent_jobs={})
self._is_handler = None
default_resubmits = []
default_resubmit_condition = self.app.config.default_job_resubmission_condition
@@ -159,10 +168,10 @@ class JobConfiguration(object, ConfiguresHandlers):
tree = load(job_config_file)
self.__parse_job_conf_xml(tree)
except IOError:
log.warning('Job configuration "%s" does not exist, using legacy'
' job configuration from Galaxy config file "%s" instead'
% (self.app.config.job_config_file, self.app.config.config_file))
self.__parse_job_conf_legacy()
log.warning('Job configuration "%s" does not exist, using default'
' job configuration (this server will run jobs)',
self.app.config.job_config_file)
self.__set_default_job_conf()
except Exception as e:
raise config_exception(e, job_config_file)
@@ -203,11 +212,21 @@ class JobConfiguration(object, ConfiguresHandlers):
handlers_conf = root.find('handlers')
self._init_handlers(handlers_conf)
# Must define at least one handler to have a default.
if not self.handlers:
raise ValueError("Job configuration file defines no valid handler elements.")
# Determine the default handler(s)
self.default_handler_id = self._get_default(self.app.config, handlers_conf, list(self.handlers.keys()))
try:
self.default_handler_id = self._get_default(self.app.config, handlers_conf, list(self.handlers.keys()))
except Exception:
pass
# For tets, this may not exist
try:
base_server_name = self.app.config.base_server_name
except AttributeError:
base_server_name = self.app.config.get('base_server_name', None)
if (self.default_handler_id is None
or (len(self.handlers) == 1 and base_server_name == self.handlers.keys()[0])):
# Shortcut for compatibility with existing job confs that use the default handlers block,
# there are no defined handlers, or there's only one handler and it's this server
self.__set_default_job_handler()
# Parse destinations
destinations = root.find('destinations')
@@ -277,15 +296,6 @@ class JobConfiguration(object, ConfiguresHandlers):
total_walltime=str,
output_size=util.size_to_bytes)
self.limits = Bunch(registered_user_concurrent_jobs=None,
anonymous_user_concurrent_jobs=None,
walltime=None,
walltime_delta=None,
total_walltime={},
output_size=None,
destination_user_concurrent_jobs={},
destination_total_concurrent_jobs={})
# Parse job limits
limits = root.find('limits')
if limits is not None:
@@ -327,73 +337,39 @@ class JobConfiguration(object, ConfiguresHandlers):
self.handler_runner_plugins[handler_id] = []
self.handler_runner_plugins[handler_id].append(plugin.get('id'))
def __parse_job_conf_legacy(self):
"""Loads the old-style job configuration from options in the galaxy config file (by default, config/galaxy.ini).
"""
log.debug('Loading job configuration from %s' % self.app.config.config_file)
# Always load local
self.runner_plugins = [dict(id='local', load='local', workers=self.app.config.local_job_queue_workers)]
def __set_default_job_conf(self):
# Run jobs locally
self.runner_plugins = [dict(id='local', load='local', workers=DEFAULT_LOCAL_WORKERS)]
# Load tasks if configured
if self.app.config.use_tasked_jobs:
self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers))
for runner in self.app.config.start_job_runners:
self.runner_plugins.append(dict(id=runner, load=runner, workers=self.app.config.cluster_job_queue_workers))
self.runner_plugins.append(dict(id='tasks', load='tasks', workers=DEFAULT_LOCAL_WORKERS))
# Set the handlers
for id in self.app.config.job_handlers:
self.handlers[id] = (id,)
self.handlers['default_job_handlers'] = self.app.config.default_job_handlers
self.default_handler_id = 'default_job_handlers'
# Set tool handler configs
for id, tool_handlers in self.app.config.tool_handlers.items():
self.tools[id] = list()
for handler_config in tool_handlers:
# rename the 'name' key to 'handler'
handler_config['handler'] = handler_config.pop('name')
self.tools[id].append(JobToolConfiguration(**handler_config))
# Set tool runner configs
for id, tool_runners in self.app.config.tool_runners.items():
# Might have been created in the handler parsing above
if id not in self.tools:
self.tools[id] = list()
for runner_config in tool_runners:
url = runner_config['url']
if url not in self.destinations:
# Create a new "legacy" JobDestination - it will have its URL converted to a destination params once the appropriate plugin has loaded
self.destinations[url] = (JobDestination(id=url, runner=url.split(':', 1)[0], url=url, legacy=True, converted=False),)
for tool_conf in self.tools[id]:
if tool_conf.params == runner_config.get('params', {}):
tool_conf['destination'] = url
break
else:
# There was not an existing config (from the handlers section) with the same params
# rename the 'url' key to 'destination'
runner_config['destination'] = runner_config.pop('url')
self.tools[id].append(JobToolConfiguration(**runner_config))
self.destinations[self.app.config.default_cluster_job_runner] = (JobDestination(id=self.app.config.default_cluster_job_runner,
runner=self.app.config.default_cluster_job_runner.split(':', 1)[0],
url=self.app.config.default_cluster_job_runner,
legacy=True,
converted=False),)
self.default_destination_id = self.app.config.default_cluster_job_runner
# Set the job limits
self.limits = Bunch(registered_user_concurrent_jobs=self.app.config.registered_user_job_limit,
anonymous_user_concurrent_jobs=self.app.config.anonymous_user_job_limit,
walltime=self.app.config.job_walltime,
walltime_delta=self.app.config.job_walltime_delta,
total_walltime={},
output_size=self.app.config.output_size_limit,
destination_user_concurrent_jobs={},
destination_total_concurrent_jobs={})
self.__set_default_job_handler()
# Set the destination
self.default_destination_id = 'local'
self.destinations['local'] = [JobDestination(id='local', runner='local')]
log.debug('Done loading job configuration')
def __set_default_job_handler(self):
# Called when self.default_handler_id is None
if self.app.application_stack.has_pool(self.app.application_stack.pools.JOB_HANDLERS):
if (self.app.application_stack.in_pool(self.app.application_stack.pools.JOB_HANDLERS)):
log.info("Found job handler pool managed by application stack, this server (%s) is a member of pool: "
"%s", self.app.config.server_name, self.app.application_stack.pools.JOB_HANDLERS)
self._is_handler = True
else:
log.info("Found job handler pool managed by application stack, this server (%s) will submit jobs to "
"pool: %s", self.app.config.server_name, self.app.application_stack.pools.JOB_HANDLERS)
else:
log.info('Did not find job handler pool managed by application stack, this server (%s) will handle jobs '
'submitted to it', self.app.config.server_name)
self._is_handler = True
self.app.application_stack.register_postfork_function(self.make_self_default_handler)
def make_self_default_handler(self):
self.default_handler_id = self.app.config.server_name
self.handlers[self.app.config.server_name] = [self.app.config.server_name]
def get_tool_resource_xml(self, tool_id, tool_type):
""" Given a tool id, return XML elements describing parameters to
insert into job resources.
@@ -1311,15 +1287,9 @@ class JobWrapper(object, HasResourceParameters):
dataset.metadata.from_JSON_dict(output_filename, path_rewriter=path_rewriter)
try:
assert context.get('line_count', None) is not None
if (not dataset.datatype.composite_type and dataset.dataset.is_multi_byte()) or self.tool.is_multi_byte:
dataset.set_peek(line_count=context['line_count'], is_multi_byte=True)
else:
dataset.set_peek(line_count=context['line_count'])
dataset.set_peek(line_count=context['line_count'])
except Exception:
if (not dataset.datatype.composite_type and dataset.dataset.is_multi_byte()) or self.tool.is_multi_byte:
dataset.set_peek(is_multi_byte=True)
else:
dataset.set_peek()
dataset.set_peek()
else:
# Handle an empty dataset.
dataset.blurb = "empty"
lib/galaxy/jobs/handler.py
+34 -1
View File
@@ -27,6 +27,7 @@ from galaxy.jobs import (
)
from galaxy.jobs.mapper import JobNotReadyException
from galaxy.util.monitors import Monitors
from galaxy.web.stack.message import JobHandlerMessage
log = logging.getLogger(__name__)
@@ -50,6 +51,7 @@ class JobHandler(object):
def start(self):
self.job_queue.start()
self.job_stop_queue.start()
def shutdown(self):
self.job_queue.shutdown()
@@ -90,10 +92,14 @@ class JobHandlerQueue(Monitors, object):
"""
Starts the JobHandler's thread after checking for any unhandled jobs.
"""
log.debug('Handler queue starting for jobs assigned to handler: %s', self.app.config.server_name)
# Recover jobs at startup
self.__check_jobs_at_startup()
# Start the queue
self.monitor_thread.start()
# The stack code is initialized in the application
JobHandlerMessage().bind_default_handler(self, '_handle_message')
self.app.application_stack.register_message_handler(self._handle_message, name=JobHandlerMessage.target)
log.info("job handler queue started")
def job_wrapper(self, job, use_persisted_destination=False):
@@ -652,11 +658,30 @@ class JobHandlerQueue(Monitors, object):
return JOB_WAIT
return JOB_READY
def _handle_setup_msg(self, job_id=None):
job = self.sa_session.query(model.Job).get(job_id)
if job.handler is None:
job.handler = self.app.config.server_name
self.sa_session.add(job)
self.sa_session.flush()
# If not tracking jobs in the database
self.put(job.id, job.tool_id)
else:
log.warning("(%s) Handler '%s' received setup message but handler '%s' is already assigned, ignoring", job.id, self.app.config.server_name, job.handler)
def put(self, job_id, tool_id):
"""Add a job to the queue (by job identifier)"""
if not self.track_jobs_in_database:
self.queue.put((job_id, tool_id))
self.sleeper.wake()
else:
# Workflow invocations farmed out to workers will submit jobs through here. If a handler is unassigned, we
# will submit for one, or else claim it ourself. TODO: This should be moved to a higher level as it's now
# implemented here and in MessageJobQueue
job = self.sa_session.query(model.Job).get(job_id)
if job.handler is None and self.app.application_stack.has_pool(self.app.application_stack.pools.JOB_HANDLERS):
msg = JobHandlerMessage(task='setup', job_id=job_id)
self.app.application_stack.send_message(self.app.application_stack.pools.JOB_HANDLERS, msg)
def shutdown(self):
"""Attempts to gracefully shut down the worker thread"""
@@ -668,6 +693,9 @@ class JobHandlerQueue(Monitors, object):
self.stop_monitoring()
if not self.app.config.track_jobs_in_database:
self.queue.put(self.STOP_SIGNAL)
# A message could still be received while shutting down, should be ok since they will be picked up on next startup.
self.app.application_stack.deregister_message_handler(name=JobHandlerMessage.target)
self.sleeper.wake()
self.shutdown_monitor()
log.info("job handler queue stopped")
self.dispatcher.shutdown()
@@ -695,7 +723,12 @@ class JobHandlerStopQueue(Monitors):
self.waiting = []
name = "JobHandlerStopQueue.monitor_thread"
self._init_monitor_thread(name, start=True, config=app.config)
self._init_monitor_thread(name, config=app.config)
log.info("job handler stop queue started")
def start(self):
# Start the queue
self.monitor_thread.start()
log.info("job handler stop queue started")
def monitor(self):
lib/galaxy/jobs/manager.py
+46 -5
View File
@@ -4,7 +4,11 @@ Top-level Galaxy job manager, moves jobs to handler(s)
import logging
from sqlalchemy.sql.expression import null
from galaxy.jobs import handler, NoopQueue
from galaxy.model import Job
from galaxy.web.stack.message import JobHandlerMessage
log = logging.getLogger(__name__)
@@ -19,15 +23,19 @@ class JobManager(object):
def __init__(self, app):
self.app = app
if self.app.is_job_handler():
log.debug("Starting job handler")
self.job_lock = False
if self.app.is_job_handler:
log.debug("Initializing job handler")
self.job_handler = handler.JobHandler(app)
self.job_queue = self.job_handler.job_queue
self.job_stop_queue = self.job_handler.job_stop_queue
elif app.application_stack.has_pool(app.application_stack.pools.JOB_HANDLERS):
log.debug("Initializing job handler messaging interface")
self.job_handler = MessageJobHandler(app)
self.job_stop_queue = NoopQueue()
else:
self.job_handler = NoopHandler()
self.job_queue = self.job_stop_queue = NoopQueue()
self.job_lock = False
self.job_stop_queue = NoopQueue()
self.job_queue = self.job_handler.job_queue
def start(self):
self.job_handler.start()
@@ -46,3 +54,36 @@ class NoopHandler(object):
def shutdown(self, *args):
pass
class MessageJobHandler(NoopHandler):
"""
Implements the JobHandler interface but just to send setup messages on startup
TODO: It should be documented that starting two Galaxy uWSGI master processes simultaneously would result in a race condition that *could* cause two handlers to pick up the same job.
The recommended config for now will be webless handlers if running more than one uWSGI (master) process
"""
def __init__(self, app):
# This runs in the web (main) process pre-fork
self.app = app
self.job_queue = MessageJobQueue(app)
self.job_stop_queue = NoopQueue()
jobs_at_startup = self.app.model.context.query(Job).enable_eagerloads(False) \
.filter((Job.state == Job.states.NEW) & (Job.handler == null())).all()
if jobs_at_startup:
log.info('No handler assigned at startup for the following jobs, will dispatch via message: %s', ', '.join([str(j.id) for j in jobs_at_startup]))
for job in jobs_at_startup:
self.job_queue.put(job.id, job.tool_id)
class MessageJobQueue(NoopQueue):
"""
Implements the JobQueue / JobStopQueue interface but only sends messages to the actual job queue
"""
def __init__(self, app):
self.app = app
def put(self, job_id, tool_id):
msg = JobHandlerMessage(task='setup', job_id=job_id)
self.app.application_stack.send_message(self.app.application_stack.pools.JOB_HANDLERS, msg)
lib/galaxy/jobs/runners/__init__.py
+11 -2
View File
@@ -83,7 +83,7 @@ class BaseJobRunner(object):
for i in range(self.nworkers):
worker = threading.Thread(name="%s.work_thread-%d" % (self.runner_name, i), target=self.run_next)
worker.setDaemon(True)
worker.start()
self.app.application_stack.register_postfork_function(worker.start)
self.work_threads.append(worker)
def run_next(self):
@@ -413,6 +413,7 @@ class JobState(object):
runner_states = Bunch(
WALLTIME_REACHED='walltime_reached',
MEMORY_LIMIT_REACHED='memory_limit_reached',
JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER='Job output not returned from cluster',
UNKNOWN_ERROR='unknown_error',
GLOBAL_WALLTIME_REACHED='global_walltime_reached',
OUTPUT_SIZE_LIMIT='output_size_limit'
@@ -612,6 +613,7 @@ class AsynchronousJobRunner(Monitors, BaseJobRunner):
# wait for the files to appear
which_try = 0
collect_output_success = True
while which_try < self.app.config.retry_job_output_collection + 1:
try:
stdout = shrink_stream_by_size(open(job_state.output_file, "r"), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
@@ -620,12 +622,19 @@ class AsynchronousJobRunner(Monitors, BaseJobRunner):
except Exception as e:
if which_try == self.app.config.retry_job_output_collection:
stdout = ''
stderr = 'Job output not returned from cluster'
stderr = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER
log.error('(%s/%s) %s: %s' % (galaxy_id_tag, external_job_id, stderr, str(e)))
collect_output_success = False
else:
time.sleep(1)
which_try += 1
if not collect_output_success:
job_state.fail_message = stderr
job_state.runner_state = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER
self.mark_as_failed(job_state)
return
try:
# This should be an 8-bit exit code, but read ahead anyway:
exit_code_str = open(job_state.exit_code_file, "r").read(32)
lib/galaxy/jobs/runners/state_handlers/resubmit.py
+1
View File
@@ -65,6 +65,7 @@ def failure(app, job_runner, job_state):
runner_state = getattr(job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR
if (runner_state not in (JobState.runner_states.WALLTIME_REACHED,
JobState.runner_states.MEMORY_LIMIT_REACHED,
JobState.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER,
JobState.runner_states.UNKNOWN_ERROR)):
# not set or not a handleable runner state
return
lib/galaxy/model/__init__.py
+2 -2
View File
@@ -2071,8 +2071,8 @@ class DatasetInstance(object):
"""Data consists of multi-byte characters"""
return self.dataset.is_multi_byte()
def set_peek(self, is_multi_byte=False):
return self.datatype.set_peek(self, is_multi_byte=is_multi_byte)
def set_peek(self):
return self.datatype.set_peek(self)
def init_meta(self, copy_from=None):
return self.datatype.init_meta(self, copy_from=copy_from)
lib/galaxy/model/metadata.py
+15 -22
View File
@@ -130,18 +130,11 @@ class MetadataCollection(object):
def element_is_set(self, name):
return bool(self.parent._metadata.get(name, False))
def get_html_by_name(self, name, **kwd):
if name in self.spec:
rval = self.spec[name].param.get_html(value=getattr(self, name), context=self, **kwd)
if rval is None:
return self.spec[name].no_value
return rval
def get_metadata_parameter(self, name, **kwd):
if name in self.spec:
html_field = self.spec[name].param.get_html_field(getattr(self, name), self, None, **kwd)
html_field.value = getattr(self, name)
return html_field
field = self.spec[name].param.get_field(getattr(self, name), self, None, **kwd)
field.value = getattr(self, name)
return field
def make_dict_copy(self, to_copy):
"""Makes a deep copy of input iterable to_copy according to self.spec"""
@@ -229,7 +222,7 @@ class MetadataParameter(object):
def __init__(self, spec):
self.spec = spec
def get_html_field(self, value=None, context=None, other_values=None, **kwd):
def get_field(self, value=None, context=None, other_values=None, **kwd):
context = context or {}
other_values = other_values or {}
return form_builder.TextField(self.spec.name, value=value)
@@ -253,9 +246,9 @@ class MetadataParameter(object):
if value:
checked = "true"
checkbox = form_builder.CheckboxField("is_" + self.spec.name, checked=checked)
return checkbox.get_html() + self.get_html_field(value=value, context=context, other_values=other_values, **kwd).get_html()
return checkbox.get_html() + self.get_field(value=value, context=context, other_values=other_values, **kwd).get_html()
else:
return self.get_html_field(value=value, context=context, other_values=other_values, **kwd).get_html()
return self.get_field(value=value, context=context, other_values=other_values, **kwd).get_html()
def to_string(self, value):
return str(value)
@@ -374,7 +367,7 @@ class SelectParameter(MetadataParameter):
value = [value]
return ",".join(map(str, value))
def get_html_field(self, value=None, context=None, other_values=None, values=None, **kwd):
def get_field(self, value=None, context=None, other_values=None, values=None, **kwd):
context = context or {}
other_values = other_values or {}
@@ -431,14 +424,14 @@ class SelectParameter(MetadataParameter):
class DBKeyParameter(SelectParameter):
def get_html_field(self, value=None, context=None, other_values=None, values=None, **kwd):
def get_field(self, value=None, context=None, other_values=None, values=None, **kwd):
context = context or {}
other_values = other_values or {}
try:
values = kwd['trans'].app.genome_builds.get_genome_build_names(kwd['trans'])
except KeyError:
pass
return super(DBKeyParameter, self).get_html_field(value, context, other_values, values, **kwd)
return super(DBKeyParameter, self).get_field(value, context, other_values, values, **kwd)
def get_html(self, value=None, context=None, other_values=None, values=None, **kwd):
context = context or {}
@@ -459,13 +452,13 @@ class RangeParameter(SelectParameter):
self.max = spec.get("max") or 1
self.step = self.spec.get("step") or 1
def get_html_field(self, value=None, context=None, other_values=None, values=None, **kwd):
def get_field(self, value=None, context=None, other_values=None, values=None, **kwd):
context = context or {}
other_values = other_values or {}
if values is None:
values = list(zip(range(self.min, self.max, self.step), range(self.min, self.max, self.step)))
return SelectParameter.get_html_field(self, value=value, context=context, other_values=other_values, values=values, **kwd)
return SelectParameter.get_field(self, value=value, context=context, other_values=other_values, values=values, **kwd)
def get_html(self, value, context=None, other_values=None, values=None, **kwd):
context = context or {}
@@ -484,14 +477,14 @@ class RangeParameter(SelectParameter):
class ColumnParameter(RangeParameter):
def get_html_field(self, value=None, context=None, other_values=None, values=None, **kwd):
def get_field(self, value=None, context=None, other_values=None, values=None, **kwd):
context = context or {}
other_values = other_values or {}
if values is None and context:
column_range = range(1, (context.columns or 0) + 1, 1)
values = list(zip(column_range, column_range))
return RangeParameter.get_html_field(self, value=value, context=context, other_values=other_values, values=values, **kwd)
return RangeParameter.get_field(self, value=value, context=context, other_values=other_values, values=values, **kwd)
def get_html(self, value, context=None, other_values=None, values=None, **kwd):
context = context or {}
@@ -532,7 +525,7 @@ class PythonObjectParameter(MetadataParameter):
return self.spec._to_string(self.spec.no_value)
return self.spec._to_string(value)
def get_html_field(self, value=None, context=None, other_values=None, **kwd):
def get_field(self, value=None, context=None, other_values=None, **kwd):
context = context or {}
other_values = other_values or {}
return form_builder.TextField(self.spec.name, value=self._to_string(value))
@@ -558,7 +551,7 @@ class FileParameter(MetadataParameter):
# We do not sanitize file names
return self.to_string(value)
def get_html_field(self, value=None, context=None, other_values=None, **kwd):
def get_field(self, value=None, context=None, other_values=None, **kwd):
context = context or {}
other_values = other_values or {}
return form_builder.TextField(self.spec.name, value=str(value.id))
lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py
+2 -2
View File
@@ -241,8 +241,8 @@ class DatasetInstance(object):
"""Saves the data on the disc"""
self.datatype.set_raw_data(self, data)
def set_peek(self, is_multi_byte=False):
return self.datatype.set_peek(self, is_multi_byte=is_multi_byte)
def set_peek(self):
return self.datatype.set_peek(self)
def init_meta(self, copy_from=None):
return self.datatype.init_meta(self, copy_from=copy_from)
lib/galaxy/model/orm/scripts.py
+20 -15
View File
@@ -2,14 +2,15 @@
Code to support database helper scripts (create_db.py, manage_db.py, etc...).
"""
import logging
import os.path
from galaxy.util import listify
from galaxy.util.path import get_ext
from galaxy.util.properties import find_config_file, load_app_properties
log = logging.getLogger(__name__)
DEFAULT_CONFIG_FILE = 'config/galaxy.ini'
DEFAULT_CONFIG_NAMES = ['galaxy', 'universe_wsgi']
DEFAULT_CONFIG_PREFIX = ''
DEFAULT_DATABASE = 'galaxy'
@@ -17,15 +18,19 @@ DATABASE = {
"galaxy":
{
'repo': 'lib/galaxy/model/migrate',
'old_config_files': ['universe_wsgi.ini'],
'default_sqlite_file': './database/universe.sqlite',
'config_override': 'GALAXY_CONFIG_',
},
"tools":
{
'repo': 'lib/tool_shed/galaxy_install/migrate',
'default_sqlite_file': './database/universe.sqlite',
'config_override': 'GALAXY_CONFIG_',
},
"tool_shed":
{
'repo': 'lib/galaxy/webapps/tool_shed/model/migrate',
'config_file': 'config/tool_shed.yml',
'old_config_files': ['config/tool_shed.ini', 'tool_shed_wsgi.ini'],
'config_names': ['tool_shed', 'tool_shed_wsgi'],
'default_sqlite_file': './database/community.sqlite',
'config_override': 'TOOL_SHED_CONFIG_',
'config_section': 'tool_shed',
@@ -33,7 +38,6 @@ DATABASE = {
"install":
{
'repo': 'lib/galaxy/model/tool_shed_install/migrate',
'old_config_files': ['universe_wsgi.ini'],
'config_prefix': 'install_',
'default_sqlite_file': './database/install.sqlite',
'config_override': 'GALAXY_INSTALL_CONFIG_',
@@ -41,14 +45,14 @@ DATABASE = {
}
def read_config_file_arg(argv, default, old_defaults, cwd=None):
config_file = None
def read_config_file_arg(argv, config_names, cwd=None):
if '-c' in argv:
pos = argv.index('-c')
argv.pop(pos)
config_file = argv.pop(pos)
old_defaults = listify(old_defaults)
return find_config_file(default, old_defaults, config_file, cwd=cwd)
return argv.pop(pos)
if cwd:
cwd = [cwd, os.path.join(cwd, 'config')]
return find_config_file(config_names, dirs=cwd)
def get_config(argv, cwd=None):
@@ -57,6 +61,7 @@ def get_config(argv, cwd=None):
>>> import os
>>> from ConfigParser import SafeConfigParser
>>> from shutil import rmtree
>>> from tempfile import mkdtemp
>>> config_dir = mkdtemp()
>>> os.makedirs(os.path.join(config_dir, 'config'))
@@ -77,6 +82,7 @@ def get_config(argv, cwd=None):
'sqlite:///moo.sqlite?isolation_level=IMMEDIATE'
>>> config['repo']
'lib/galaxy/model/migrate'
>>> rmtree(config_dir)
"""
if argv and (argv[-1] in DATABASE):
database = argv.pop() # database name tool_shed, galaxy, or install.
@@ -84,14 +90,13 @@ def get_config(argv, cwd=None):
database = 'galaxy'
database_defaults = DATABASE[database]
default = database_defaults.get('config_file', DEFAULT_CONFIG_FILE)
old_defaults = database_defaults.get('old_config_files')
config_file = read_config_file_arg(argv, default, old_defaults, cwd=cwd)
config_names = database_defaults.get('config_names', DEFAULT_CONFIG_NAMES)
config_file = read_config_file_arg(argv, config_names, cwd=cwd)
repo = database_defaults['repo']
config_prefix = database_defaults.get('config_prefix', DEFAULT_CONFIG_PREFIX)
config_override = database_defaults.get('config_override', 'GALAXY_CONFIG_')
default_sqlite_file = database_defaults['default_sqlite_file']
if config_file.endswith(".yml") or config_file.endswith(".yml.sample"):
if not config_file or get_ext(config_file, ignore='sample') == 'yaml':
config_section = database_defaults.get('config_section', None)
else:
# An .ini file - just let load_app_properties find app:main.
lib/galaxy/tools/actions/__init__.py
+1 -1
View File
@@ -510,7 +510,7 @@ class DefaultToolAction(object):
trans.response.send_redirect(url_for(controller='tool_runner', action='redirect', redirect_url=redirect_url))
else:
# Put the job in the queue if tracking in memory
app.job_queue.put(job.id, job.tool_id)
app.job_manager.job_queue.put(job.id, job.tool_id)
trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
return job, out_data
lib/galaxy/tools/actions/history_imp_exp.py
+2 -2
View File
@@ -59,7 +59,7 @@ class ImportHistoryToolAction(ToolAction):
trans.sa_session.flush()
# Queue the job for execution
trans.app.job_queue.put(job.id, tool.id)
trans.app.job_manager.job_queue.put(job.id, tool.id)
trans.log_event("Added import history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
return job, odict()
@@ -138,7 +138,7 @@ class ExportHistoryToolAction(ToolAction):
trans.sa_session.flush()
# Queue the job for execution
trans.app.job_queue.put(job.id, tool.id)
trans.app.job_manager.job_queue.put(job.id, tool.id)
trans.log_event("Added export history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
return job, odict()
lib/galaxy/tools/actions/metadata.py
+1 -1
View File
@@ -103,7 +103,7 @@ class SetMetadataToolAction(ToolAction):
sa_session.flush()
# Queue the job for execution
app.job_queue.put(job.id, tool.id)
app.job_manager.job_queue.put(job.id, tool.id)
# FIXME: need to add event logging to app and log events there rather than trans.
# trans.log_event( "Added set external metadata job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id )
lib/galaxy/tools/actions/model_operations.py
+1 -1
View File
@@ -61,7 +61,7 @@ class ModelOperationToolAction(DefaultToolAction):
trans.sa_session.flush() # ensure job.id are available
# Queue the job for execution
# trans.app.job_queue.put( job.id, tool.id )
# trans.app.job_manager.job_queue.put( job.id, tool.id )
# trans.log_event( "Added database job action to the job queue, id: %s" % str(job.id), tool_id=job.tool_id )
log.info("Calling produce_outputs, tool is %s" % tool)
return job, out_data
lib/galaxy/tools/actions/upload_common.py
+1 -1
View File
@@ -516,7 +516,7 @@ def create_job(trans, params, tool, json_file_path, data_list, folder=None, hist
trans.sa_session.flush()
# Queue the job for execution
trans.app.job_queue.put(job.id, job.tool_id)
trans.app.job_manager.job_queue.put(job.id, job.tool_id)
trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
output = odict()
for i, v in enumerate(data_list):
lib/galaxy/tools/error_reports/plugins/sentry.py
+13 -3
View File
@@ -31,11 +31,21 @@ class SentryPlugin(ErrorPlugin):
self.app = kwargs['app']
self.verbose = string_as_bool(kwargs.get('verbose', False))
self.user_submission = string_as_bool(kwargs.get('user_submission', False))
self.custom_dsn = kwargs.get('custom_dsn', None)
self.sentry = None
# Use the built in one by default
if hasattr(self.app, 'sentry_client'):
self.sentry = self.app.sentry_client
# if they've set a custom one, override.
if self.custom_dsn:
import raven
self.sentry = raven.Client(self.custom_dsn)
def submit_report(self, dataset, job, tool, **kwargs):
"""Submit the error report to sentry
"""
if self.app.sentry_client:
if self.sentry_client:
user = job.get_user()
extra = {
'info': job.info,
@@ -64,7 +74,7 @@ class SentryPlugin(ErrorPlugin):
error_message = ERROR_TEMPLATE.format(**extra)
# Update context with user information in a sentry-specific manner
self.app.sentry_client.context.merge({
self.sentry_client.context.merge({
# User information here also places email links + allows seeing
# a list of affected users in the tags/filtering.
'user': {
@@ -83,7 +93,7 @@ class SentryPlugin(ErrorPlugin):
})
# Send the message, using message because
response = self.app.sentry_client.capture(
response = self.sentry_client.capture(
'raven.events.Message',
tags={
'tool_id': job.tool_id,
lib/galaxy/tools/imp_exp/imp_history_from_archive.xml
+1 -1
View File
@@ -1,7 +1,7 @@
<tool id="__IMPORT_HISTORY__" name="Import History" version="0.1" tool_type="import_history">
<type class="ImportHistoryTool" module="galaxy.tools"/>
<action module="galaxy.tools.actions.history_imp_exp" class="ImportHistoryToolAction"/>
<command interpreter="python">unpack_tar_gz_archive.py "${ str( $__ARCHIVE_SOURCE__ ).encode( 'base64' ) }" "${ str( $__DEST_DIR__ ).encode( 'base64' ) }" --$__ARCHIVE_TYPE__ --encoded</command>
<command>python '$__tool_directory__/unpack_tar_gz_archive.py' '${ str( $__ARCHIVE_SOURCE__ ).encode( 'base64' ) }' '${ str( $__DEST_DIR__ ).encode( 'base64' ) }' --$__ARCHIVE_TYPE__ --encoded</command>
<inputs>
<param name="__ARCHIVE_SOURCE__" type="text">
<sanitizer sanitize="False"/>
lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py
+2 -2
View File
@@ -49,7 +49,7 @@ def check_archive(archive_file, dest_dir):
Ensure that a tar archive has no absolute paths or relative paths outside
the archive.
"""
with tarfile.open(archive_file, mode='r:gz') as archive_fp:
with tarfile.open(archive_file, mode='r') as archive_fp:
for arc_path in archive_fp.getnames():
assert os.path.normpath(
os.path.join(
@@ -64,7 +64,7 @@ def unpack_archive(archive_file, dest_dir):
"""
Unpack a tar and/or gzipped archive into a destination directory.
"""
archive_fp = tarfile.open(archive_file, mode='r:gz')
archive_fp = tarfile.open(archive_file, mode='r')
archive_fp.extractall(path=dest_dir)
archive_fp.close()
lib/galaxy/tools/verify/__init__.py
+59 -62
View File
@@ -168,77 +168,74 @@ def files_diff(file1, file2, attributes=None):
if (line.startswith('+') and not line.startswith('+++')) or (line.startswith('-') and not line.startswith('---')):
count += 1
return count
if not filecmp.cmp(file1, file2):
files_differ = False
if attributes is None:
attributes = {}
decompress = attributes.get("decompress", None)
if not decompress:
local_file = open(file1, 'U').readlines()
history_data = open(file2, 'U').readlines()
if decompress:
# None means all compressed formats are allowed
compressed_formats = None
else:
local_file = get_fileobj(file1, 'U').readlines()
history_data = get_fileobj(file2, 'U').readlines()
compressed_formats = []
is_pdf = False
try:
local_file = get_fileobj(file1, 'U', compressed_formats=compressed_formats).readlines()
history_data = get_fileobj(file2, 'U', compressed_formats=compressed_formats).readlines()
except UnicodeDecodeError:
if file1.endswith('.pdf') or file2.endswith('.pdf'):
is_pdf = True
local_file = open(file1, 'rb').readlines()
history_data = open(file2, 'rb').readlines()
else:
raise AssertionError("Binary data detected, not displaying diff")
if attributes.get('sort', False):
history_data.sort()
# Why even bother with the check loop below, why not just use the diff output? This seems wasteful.
if len(local_file) == len(history_data):
for i in range(len(history_data)):
if local_file[i].rstrip('\r\n') != history_data[i].rstrip('\r\n'):
files_differ = True
break
else:
files_differ = True
if files_differ:
allowed_diff_count = int(attributes.get('lines_diff', 0))
diff = list(difflib.unified_diff(local_file, history_data, "local_file", "history_data"))
diff_lines = get_lines_diff(diff)
if diff_lines > allowed_diff_count:
if 'GALAXY_TEST_RAW_DIFF' in os.environ:
diff_slice = diff
allowed_diff_count = int(attributes.get('lines_diff', 0))
diff = list(difflib.unified_diff(local_file, history_data, "local_file", "history_data"))
diff_lines = get_lines_diff(diff)
if diff_lines > allowed_diff_count:
if 'GALAXY_TEST_RAW_DIFF' in os.environ:
diff_slice = diff
else:
if len(diff) < 60:
diff_slice = diff[0:40]
else:
if len(diff) < 60:
diff_slice = diff[0:40]
else:
diff_slice = diff[:25] + ["********\n", "*SNIP *\n", "********\n"] + diff[-25:]
# FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff
# due to unknown desired behavior when used in conjunction with a non-zero lines_diff
# PDF forgiveness can probably be handled better by not special casing by __extension__ here
# and instead using lines_diff or a regular expression matching
# or by creating and using a specialized pdf comparison function
if file1.endswith('.pdf') or file2.endswith('.pdf'):
# PDF files contain creation dates, modification dates, ids and descriptions that change with each
# new file, so we need to handle these differences. As long as the rest of the PDF file does
# not differ we're ok.
valid_diff_strs = ['description', 'createdate', 'creationdate', 'moddate', 'id', 'producer', 'creator']
valid_diff = False
invalid_diff_lines = 0
for line in diff_slice:
# Make sure to lower case strings before checking.
line = line.lower()
# Diff lines will always start with a + or - character, but handle special cases: '--- local_file \n', '+++ history_data \n'
if (line.startswith('+') or line.startswith('-')) and line.find('local_file') < 0 and line.find('history_data') < 0:
for vdf in valid_diff_strs:
if line.find(vdf) < 0:
valid_diff = False
else:
valid_diff = True
# Stop checking as soon as we know we have a valid difference
break
if not valid_diff:
invalid_diff_lines += 1
log.info('## files diff on %s and %s lines_diff=%d, found diff = %d, found pdf invalid diff = %d' % (file1, file2, allowed_diff_count, diff_lines, invalid_diff_lines))
if invalid_diff_lines > allowed_diff_count:
# Print out diff_slice so we can see what failed
log.info("###### diff_slice ######")
raise AssertionError("".join(diff_slice))
else:
log.info('## files diff on %s and %s lines_diff=%d, found diff = %d' % (file1, file2, allowed_diff_count, diff_lines))
for line in diff_slice:
for char in line:
if ord(char) > 128:
raise AssertionError("Binary data detected, not displaying diff")
diff_slice = diff[:25] + ["********\n", "*SNIP *\n", "********\n"] + diff[-25:]
# FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff
# due to unknown desired behavior when used in conjunction with a non-zero lines_diff
# PDF forgiveness can probably be handled better by not special casing by __extension__ here
# and instead using lines_diff or a regular expression matching
# or by creating and using a specialized pdf comparison function
if is_pdf:
# PDF files contain creation dates, modification dates, ids and descriptions that change with each
# new file, so we need to handle these differences. As long as the rest of the PDF file does
# not differ we're ok.
valid_diff_strs = ['description', 'createdate', 'creationdate', 'moddate', 'id', 'producer', 'creator']
valid_diff = False
invalid_diff_lines = 0
for line in diff_slice:
# Make sure to lower case strings before checking.
line = line.lower()
# Diff lines will always start with a + or - character, but handle special cases: '--- local_file \n', '+++ history_data \n'
if (line.startswith('+') or line.startswith('-')) and line.find('local_file') < 0 and line.find('history_data') < 0:
for vdf in valid_diff_strs:
if line.find(vdf) < 0:
valid_diff = False
else:
valid_diff = True
# Stop checking as soon as we know we have a valid difference
break
if not valid_diff:
invalid_diff_lines += 1
log.info('## files diff on %s and %s lines_diff=%d, found diff = %d, found pdf invalid diff = %d' % (file1, file2, allowed_diff_count, diff_lines, invalid_diff_lines))
if invalid_diff_lines > allowed_diff_count:
# Print out diff_slice so we can see what failed
log.info("###### diff_slice ######")
raise AssertionError("".join(diff_slice))
else:
log.info('## files diff on %s and %s lines_diff=%d, found diff = %d' % (file1, file2, allowed_diff_count, diff_lines))
raise AssertionError("".join(diff_slice))
def files_re_match(file1, file2, attributes=None):
lib/galaxy/util/compression_utils.py
+25 -12
View File
@@ -1,4 +1,5 @@
import gzip
import io
import sys
import zipfile
@@ -9,32 +10,44 @@ from .checkers import (
if sys.version_info < (3, 3):
import bz2file as bz2
gzip.GzipFile.read1 = gzip.GzipFile.read # workaround for https://bugs.python.org/issue12591
else:
import bz2
def get_fileobj(filename, mode="r", gzip_only=False, bz2_only=False, zip_only=False):
def get_fileobj(filename, mode="r", compressed_formats=None):
"""
Returns a fileobj. If the file is compressed, return appropriate file reader.
Returns a fileobj. If the file is compressed, return an appropriate file
reader. In text mode, always use 'utf-8' encoding.
:param filename: path to file that should be opened
:param mode: mode to pass to opener
:param gzip_only: only open file if file is gzip compressed or not compressed
:param bz2_only: only open file if file is bz2 compressed or not compressed
:param zip_only: only open file if file is zip compressed or not compressed
:param compressed_formats: list of allowed compressed file formats among
'bz2', 'gzip' and 'zip'. If left to None, all 3 formats are allowed
"""
if compressed_formats is None:
compressed_formats = ['bz2', 'gzip', 'zip']
# Remove 't' from mode, which may cause an error for compressed files
mode = mode.replace('t', '')
# the various compression readers don't support 'U' mode,
# so we open in 'r'.
if mode == 'U':
cmode = 'r'
else:
cmode = mode
if not bz2_only and not zip_only and is_gzip(filename):
return gzip.GzipFile(filename, cmode)
if not gzip_only and not zip_only and is_bz2(filename):
return bz2.BZ2File(filename, cmode)
if not bz2_only and not gzip_only and zipfile.is_zipfile(filename):
if 'gzip' in compressed_formats and is_gzip(filename):
fh = gzip.GzipFile(filename, cmode)
elif 'bz2' in compressed_formats and is_bz2(filename):
fh = bz2.BZ2File(filename, cmode)
elif 'zip' in compressed_formats and zipfile.is_zipfile(filename):
# Return fileobj for the first file in a zip file.
with zipfile.ZipFile(filename, cmode) as zh:
return zh.open(zh.namelist()[0], cmode)
return open(filename, mode)
fh = zh.open(zh.namelist()[0], cmode)
elif 'b' in mode:
return open(filename, mode)
else:
return io.open(filename, mode, encoding='utf-8')
if 'b' not in mode:
return io.TextIOWrapper(fh, encoding='utf-8')
else:
return fh
lib/galaxy/util/facts.py
+65
View File
@@ -0,0 +1,65 @@
"""Return various facts for string formatting.
"""
import socket
from collections import MutableMapping
from six import string_types
class Facts(MutableMapping):
"""A dict-like object that evaluates values at access time."""
def __init__(self, config=None, **kwargs):
self.__dict__ = {}
self.__set_defaults(config)
self.__set_config(config)
self.__dict__.update(dict(**kwargs))
def __set_defaults(self, config):
# config here may be a Galaxy config object, or it may just be a dict
defaults = {
'server_name': config.base_server_name,
'server_id': None,
'instance_id': None,
'pool_name': None,
'fqdn': lambda: socket.getfqdn(),
'hostname': lambda: socket.gethostname().split('.', 1)[0],
}
self.__dict__.update(defaults)
def __set_config(self, config):
if config is not None:
for name in dir(config):
if not name.startswith('_') and isinstance(getattr(config, name), string_types):
self.__dict__['config_' + name] = lambda name=name: getattr(config, name)
def __getitem__(self, key):
item = self.__dict__.__getitem__(key)
if callable(item):
return item()
else:
return item
# Other methods pass through to the corresponding dict methods
def __setitem__(self, key, value):
return self.__dict__.__setitem__(key, value)
def __delitem__(self, key):
return self.__dict__.__delitem__(key)
def __iter__(self):
return self.__dict__.__iter__()
def __len__(self):
return self.__dict__.__len__()
def __str__(self):
return self.__dict__.__str__()
def __repr__(self):
return self.__dict__.__repr__()
def get_facts(config=None, **kwargs):
return Facts(config=config, **kwargs)
lib/galaxy/util/handlers.py
+9 -3
View File
@@ -9,6 +9,7 @@ import logging
import os
import random
log = logging.getLogger(__name__)
@@ -93,16 +94,19 @@ class ConfiguresHandlers:
rval.append(elem)
return rval
def is_handler(self, server_name):
"""Given a server name, indicate whether the server is a handler.
@property
def is_handler(self):
"""Indicate whether the current server is a handler.
:param server_name: The name to check
:type server_name: str
:return: bool
"""
if self._is_handler is not None:
return self._is_handler
for collection in self.handlers.values():
if server_name in collection:
if self.app.config.server_name in collection:
return True
return False
@@ -128,6 +132,8 @@ class ConfiguresHandlers:
:returns: str -- A valid job handler ID.
"""
if id_or_tag is None and self.default_handler_id is None:
return None
if id_or_tag is None:
id_or_tag = self.default_handler_id
return self._get_single_item(self.handlers[id_or_tag], index=index)
lib/galaxy/util/path/__init__.py
+109 -1
View File
@@ -8,6 +8,7 @@ from functools import partial
from itertools import starmap
from operator import getitem
from os import (
extsep,
makedirs,
walk,
)
@@ -22,7 +23,7 @@ from os.path import (
relpath,
)
from six import string_types
from six import iteritems, string_types
from six.moves import filterfalse, map, zip
@@ -97,6 +98,92 @@ def unsafe_walk(path, whitelist=None):
return filterfalse(partial(safe_contains, path, whitelist=whitelist), __walk(abspath(path)))
def joinext(root, ext):
"""
Roughly the reverse of os.path.splitext.
:type root: string
:param root: part of the filename before the extension
:type root: string
:param ext: the extension
:rtype: string
:returns: ``root`` joined with ``ext`` separated by a single ``os.extsep``
"""
return extsep.join([root.rstrip(extsep), ext.lstrip(extsep)])
def has_ext(path, ext, aliases=False, ignore=None):
"""
Determine whether ``path`` has extension ``ext``
:type path: string
:param path: Path to check
:type ext: string
:param ext: Extension to check
:type aliases: bool
:param aliases: Check any known aliases for the given extension
:type ignore: string
:param ignore: Ignore this extension at the end of the path (e.g. ``sample``)
:rtype: bool
:returns: ``True`` if path is a YAML file, ``False`` otherwise.
"""
ext = __ext_strip_sep(ext)
root, _ext = __splitext_ignore(path, ignore=ignore)
if aliases:
return _ext in extensions[ext]
else:
return _ext == ext
def get_ext(path, ignore=None, canonicalize=True):
"""
Return the extension of ``path``
:type path: string
:param path: Path to check
:type ignore: string
:param ignore: Ignore this extension at the end of the path (e.g. ``sample``)
:type canonicalize: bool
:param canonicalize: If the extension is known to this module, return the canonicalized extension instead of the
file's actual extension
:rtype: string
"""
root, ext = __splitext_ignore(path, ignore=ignore)
if canonicalize:
try:
ext = extensions.canonicalize(ext)
except KeyError:
pass # should do something else here?
return ext
class Extensions(dict):
"""Mappings for extension aliases.
A dict-like object that returns values for keys that are not mapped if the key can be found in any of the dict's
values (which should be sequence types).
The first item in the sequence should match the key and is the "canonicalization".
"""
def __missing__(self, key):
for k, v in iteritems(self):
if key in v:
self[key] = v
return v
raise KeyError(key)
def canonicalize(self, ext):
# shouldn't raise an IndexError because it should raise a KeyError first
return self[ext][0]
extensions = Extensions({
'ini': ['ini'],
'json': ['json'],
'yaml': ['yaml', 'yml'],
})
def __listify(item):
"""A non-splitting version of :func:`galaxy.util.listify`.
"""
@@ -124,6 +211,23 @@ def __contains(prefix, path, whitelist=None):
yield not relpath(real, wldir).startswith(pardir)
def __ext_strip_sep(ext):
return ext.lstrip(extsep)
def __splitext_no_sep(path):
return (path.rsplit(extsep, 1) + [''])[0:2]
def __splitext_ignore(path, ignore=None):
# note: unlike os.path.splitext this strips extsep from ext
ignore = map(__ext_strip_sep, __listify(ignore))
root, ext = __splitext_no_sep(path)
if ext in ignore:
root, ext = __splitext_no_sep(path)
return (root, ext)
# cross-platform support
@@ -178,6 +282,10 @@ __pathfxns__ = (
)
__all__ = (
'extensions',
'get_ext',
'has_ext',
'joinext',
'safe_contains',
'safe_makedirs',
'safe_relpath',
lib/galaxy/util/properties.py
+84 -36
View File
@@ -5,41 +5,62 @@ this should be reusable by tool shed and pulsar as well.
import os
import os.path
import sys
from functools import partial
from itertools import product, starmap
import yaml
from six import iteritems
from six import iteritems, string_types
from six.moves.configparser import ConfigParser
from galaxy.util import listify
from galaxy.util.path import extensions, has_ext, joinext
def find_config_file(default, old_defaults, explicit, cwd=None):
old_defaults = listify(old_defaults)
if cwd is not None:
default = os.path.join(cwd, default)
for i in range(len(old_defaults)):
old_defaults[i] = os.path.join(cwd, old_defaults[i])
if explicit is not None:
explicit = os.path.join(cwd, explicit)
def find_config_file(names, exts=None, dirs=None, include_samples=False):
"""Locate a config file in multiple directories, with multiple extensions.
if explicit:
if os.path.exists(explicit):
config_file = explicit
else:
raise Exception("Problem determining Galaxy's configuration - the specified configuration file cannot be found.")
else:
config_file = None
if os.path.exists(default):
config_file = default
if config_file is None:
for old_default in old_defaults:
if os.path.exists(old_default):
config_file = old_default
if config_file is None:
config_file = default + ".sample"
return config_file
>>> from shutil import rmtree
>>> from tempfile import mkdtemp
>>> def touch(d, f):
... open(os.path.join(d, f), 'w').close()
>>> def _find_config_file(*args, **kwargs):
... return find_config_file(*args, **kwargs).replace(d, '')
>>> d = mkdtemp()
>>> d1 = os.path.join(d, 'd1')
>>> d2 = os.path.join(d, 'd2')
>>> os.makedirs(d1)
>>> os.makedirs(d2)
>>> touch(d1, 'foo.ini')
>>> touch(d1, 'foo.bar')
>>> touch(d1, 'baz.ini.sample')
>>> touch(d2, 'foo.yaml')
>>> touch(d2, 'baz.yml')
>>> _find_config_file('foo', dirs=(d1, d2))
'/d1/foo.ini'
>>> _find_config_file('baz', dirs=(d1, d2))
'/d2/baz.yml'
>>> _find_config_file('baz', dirs=(d1, d2), include_samples=True)
'/d2/baz.yml'
>>> _find_config_file('baz', dirs=(d1,), include_samples=True)
'/d1/baz.ini.sample'
>>> _find_config_file('foo', dirs=(d2, d1))
'/d2/foo.yaml'
>>> find_config_file('quux', dirs=(d,))
>>> _find_config_file('foo', exts=('bar', 'ini'), dirs=(d1,))
'/d1/foo.bar'
>>> rmtree(d)
"""
found = __find_config_files(
names,
exts=exts or extensions['yaml'] + extensions['ini'],
dirs=dirs or [os.getcwd(), os.path.join(os.getcwd(), 'config')],
include_samples=include_samples,
)
if not found:
return None
# doesn't really make sense to log here but we should probably generate a warning of some kind if more than one
# config is found.
return found[0]
def load_app_properties(
@@ -56,18 +77,22 @@ def load_app_properties(
config_section = ini_section
if config_file:
if not config_file.endswith(".yml") and not config_file.endswith(".yml.sample"):
if not has_ext(config_file, 'yaml', aliases=True, ignore='sample'):
if config_section is None:
config_section = "app:main"
parser = nice_config_parser(config_file)
properties.update(dict(parser.items(config_section)))
if parser.has_section(config_section):
properties.update(dict(parser.items(config_section)))
else:
properties.update(parser.defaults())
else:
if config_section is None:
config_section = "galaxy"
with open(config_file, "r") as f:
raw_properties = yaml.safe_load(f)
properties = raw_properties[config_section] or {}
properties = __default_properties(config_file)
properties.update(raw_properties.get(config_section) or {})
override_prefix = "%sOVERRIDE_" % config_prefix
for key in os.environ:
@@ -83,11 +108,7 @@ def load_app_properties(
def nice_config_parser(path):
defaults = {
'here': os.path.dirname(os.path.abspath(path)),
'__file__': os.path.abspath(path)
}
parser = NicerConfigParser(path, defaults=defaults)
parser = NicerConfigParser(path, defaults=__default_properties(path))
parser.optionxform = str # Don't lower-case keys
with open(path) as f:
parser.read_file(f)
@@ -149,4 +170,31 @@ class NicerConfigParser(ConfigParser):
raise
def __get_all_configs(dirs, names):
return list(filter(os.path.exists, starmap(os.path.join, product(dirs, names))))
def __find_config_files(names, exts=None, dirs=None, include_samples=False):
sample_names = []
if isinstance(names, string_types):
names = [names]
if not dirs:
dirs = [os.getcwd()]
if exts:
# add exts to names, converts back into a list because it's going to be small and we might consume names twice
names = list(starmap(joinext, product(names, exts)))
if include_samples:
sample_names = map(partial(joinext, ext='sample'), names)
# check for all names in each dir before moving to the next dir. could do it the other way around but that makes
# less sense to me.
return __get_all_configs(dirs, names) or __get_all_configs(dirs, sample_names)
def __default_properties(path):
return {
'here': os.path.dirname(os.path.abspath(path)),
'__file__': os.path.abspath(path)
}
__all__ = ('find_config_file', 'load_app_properties', 'NicerConfigParser')
lib/galaxy/web/form_builder.py
+9 -2
View File
@@ -8,7 +8,11 @@ from cgi import escape
from six import string_types
from galaxy.util import restore_text, unicodify
from galaxy.util import (
asbool,
restore_text,
unicodify
)
log = logging.getLogger(__name__)
@@ -19,7 +23,10 @@ class BaseField(object):
self.label = label
self.value = value
self.disabled = kwds.get('disabled', False)
self.optional = kwds.get('optional', True) and kwds.get('required', 'optional') == 'optional'
if 'optional' in kwds:
self.optional = asbool(kwds.get('optional'))
else:
self.optional = kwds.get('required', 'optional') == 'optional'
self.help = kwds.get('helptext')
def get_html(self, prefix=""):
lib/galaxy/web/framework/webapp.py
+4 -11
View File
@@ -37,6 +37,7 @@ from galaxy.web.framework import (
helpers,
url_for
)
from galaxy.web.stack import get_app_kwds
log = logging.getLogger(__name__)
@@ -961,17 +962,9 @@ def build_native_uwsgi_app(paste_factory, config_section):
"""uwsgi can load paste factories with --ini-paste, but this builds non-paste uwsgi apps.
In particular these are useful with --yaml or --json for config."""
import uwsgi
uwsgi_opt = uwsgi.opt
config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
if not config_file:
# Probably loaded via --ini-paste - expect paste app.
return None
uwsgi_app = paste_factory(uwsgi.opt, load_app_kwds={
"config_file": config_file,
"config_section": config_section,
})
# TODO: just move this to a classmethod on stack?
app_kwds = get_app_kwds(config_section)
uwsgi_app = paste_factory({}, load_app_kwds=app_kwds)
return uwsgi_app
lib/galaxy/web/stack/__init__.py
+294 -21
View File
@@ -1,6 +1,6 @@
"""Web application stack operations
"""
from __future__ import print_function
from __future__ import absolute_import
import inspect
import logging
@@ -14,59 +14,297 @@ try:
except ImportError:
uwsgi = None
try:
from uwsgidecorators import postfork as uwsgi_postfork
except (AttributeError, ImportError):
uwsgi_postfork = lambda x: x # noqa: E731
if uwsgi is not None and hasattr(uwsgi, 'numproc'):
print("WARNING: This is a uwsgi process but the uwsgidecorators library"
" is unavailable. This is likely due to using an external (not"
" in Galaxy's virtualenv) uwsgi and you may experience errors. "
"HINT:\n {venv}/bin/pip install uwsgidecorators".format(
venv=os.environ.get('VIRTUAL_ENV', '/path/to/venv')))
from six import string_types
from galaxy.util.bunch import Bunch
from galaxy.util.facts import get_facts
from galaxy.util.properties import nice_config_parser
from .message import ApplicationStackMessage, ApplicationStackMessageDispatcher
from .transport import ApplicationStackTransport, UWSGIFarmMessageTransport
log = logging.getLogger(__name__)
class ApplicationStackLogFilter(logging.Filter):
def filter(self, record):
record.worker_id = None
record.mule_id = None
return True
class UWSGILogFilter(logging.Filter):
def filter(self, record):
record.worker_id = uwsgi.worker_id()
record.mule_id = uwsgi.mule_id()
return True
class ApplicationStack(object):
name = None
prohibited_middleware = frozenset()
transport_class = ApplicationStackTransport
log_filter_class = ApplicationStackLogFilter
log_format = '%(name)s %(levelname)s %(asctime)s %(message)s'
# TODO: this belongs in the pool configuration
server_name_template = '{server_name}'
default_app_name = 'main'
# used both to route jobs to a pool with this name and indicate whether or
# not a stack is using messaging for handler assignment
pools = Bunch(
JOB_HANDLERS='job-handlers',
)
@classmethod
def log_filter(cls):
return cls.log_filter_class()
@classmethod
def get_app_kwds(cls, config_section, app_name=None, for_paste_app=False):
return {}
@classmethod
def register_postfork_function(cls, f, *args, **kwargs):
f(*args, **kwargs)
def workers(self):
return []
def __init__(self, app=None, config=None):
self.app = app
self.config = config or (app and app.config)
self.running = False
def start(self):
# TODO: with a stack config the pools could be parsed here
pass
def allowed_middleware(self, middleware):
if hasattr(middleware, '__name__'):
middleware = middleware.__name__
return middleware not in self.prohibited_middleware
def workers(self):
return []
@property
def pool_name(self):
# TODO: ideally jobs would be mappable to handlers by pool name
return None
def has_pool(self, pool_name):
return False
def in_pool(self, pool_name):
return False
@property
def facts(self):
facts = get_facts(config=self.config)
facts.update({'pool_name': self.pool_name})
return facts
def set_postfork_server_name(self, app):
app.config.server_name = self.server_name_template.format(**self.facts)
log.debug('server_name set to: %s', app.config.server_name)
def register_message_handler(self, func, name=None):
pass
def deregister_message_handler(self, func=None, name=None):
pass
def send_message(self, dest, msg=None, target=None, params=None, **kwargs):
pass
def shutdown(self):
pass
class UWSGIApplicationStack(ApplicationStack):
class MessageApplicationStack(ApplicationStack):
def __init__(self, app=None, config=None):
super(MessageApplicationStack, self).__init__(app=app, config=config)
self.dispatcher = ApplicationStackMessageDispatcher()
self.transport = self.transport_class(app, stack=self, dispatcher=self.dispatcher)
def start(self):
super(MessageApplicationStack, self).start()
if not self.running:
self.transport.start()
self.running = True
def register_message_handler(self, func, name=None):
self.dispatcher.register_func(func, name)
self.transport.start_if_needed()
def deregister_message_handler(self, func=None, name=None):
self.dispatcher.deregister_func(func, name)
self.transport.stop_if_unneeded()
def send_message(self, dest, msg=None, target=None, params=None, **kwargs):
assert msg is not None or target is not None, "Either 'msg' or 'target' parameters must be set"
if not msg:
msg = ApplicationStackMessage(
target=target,
params=params,
**kwargs
)
self.transport.send_message(msg.encode(), dest)
def shutdown(self):
if self.running:
log.info('Application stack interface shutting down')
self.transport.shutdown()
self.running = False
class UWSGIApplicationStack(MessageApplicationStack):
"""Interface to the uWSGI application stack. Supports running additional webless Galaxy workers as mules. Mules
must be farmed to be communicable via uWSGI mule messaging, unfarmed mules are not supported.
Note that mules will use this as their stack class even though they start with the "webless" loading point.
"""
name = 'uWSGI'
prohibited_middleware = frozenset([
'wrap_in_static',
'EvalException',
])
transport_class = UWSGIFarmMessageTransport
log_filter_class = UWSGILogFilter
log_format = '%(name)s %(levelname)s %(asctime)s [p:%(process)s,w:%(worker_id)s,m:%(mule_id)s] [%(threadName)s] %(message)s'
server_name_template = '{server_name}.{pool_name}.{instance_id}'
postfork_functions = []
@classmethod
def get_app_kwds(cls, config_section, app_name=None):
kwds = {
'config_file': None,
'config_section': config_section,
}
uwsgi_opt = uwsgi.opt
# check for --yaml or --json uWSGI config options first
config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
# --ini and --ini-paste don't behave the same way, but this method will only be called by mules if the main
# application was loaded with --ini-paste, so we can make some assumptions, most notably, uWSGI does not have
# any way to set the app name when loading with paste.deploy:loadapp(), so hardcoding the alternate section
# name to `app:main` is fine.
if config_file is None and uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste"):
config_file = uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste")
parser = nice_config_parser(config_file)
if not parser.has_section(config_section) and parser.has_section('app:main'):
kwds['config_section'] = 'app:main'
# check for --set galaxy_config_file=<path>, this overrides whatever config file uWSGI was loaded with (which
# may not actually include a Galaxy config)
if uwsgi_opt.get("galaxy_config_file"):
config_file = uwsgi_opt.get("galaxy_config_file")
kwds['config_file'] = config_file
return kwds
@classmethod
def register_postfork_function(cls, f, *args, **kwargs):
cls.postfork_functions.append((f, args, kwargs))
if uwsgi.mule_id() == 0:
cls.postfork_functions.append((f, args, kwargs))
else:
# mules are forked from the master and run the master's postfork functions immediately before the forked
# process is replaced. that is prevented in the _do_uwsgi_postfork function, and because programmed mules
# are standalone non-forking processes, they should run postfork functions immediately
f(*args, **kwargs)
def __init__(self, app=None, config=None):
self._farms_dict = None
self._mules_list = None
super(UWSGIApplicationStack, self).__init__(app=app, config=config)
@property
def _configured_mules(self):
if self._mules_list is None:
self._mules_list = _uwsgi_configured_mules()
return self._mules_list
@property
def _is_mule(self):
return uwsgi.mule_id() > 0
@property
def _configured_farms(self):
if self._farms_dict is None:
self._farms_dict = {}
farms = uwsgi.opt.get('farm', [])
farms = [farms] if isinstance(farms, string_types) else farms
for farm in farms:
name, mules = farm.split(':', 1)
self._farms_dict[name] = [int(m) for m in mules.split(',')]
return self._farms_dict
@property
def _farms(self):
farms = []
for farm, mules in self._configured_farms.items():
if uwsgi.mule_id() in mules:
farms.append(farm)
return farms
def _mule_index_in_farm(self, farm_name):
try:
mules = self._configured_farms[farm_name]
return mules.index(uwsgi.mule_id())
except (KeyError, ValueError):
return -1
@property
def _farm_name(self):
try:
return self._farms[0]
except IndexError:
return None
@property
def instance_id(self):
if not self._is_mule:
instance_id = uwsgi.worker_id()
elif self._farm_name:
return self._mule_index_in_farm(self._farm_name) + 1
else:
instance_id = uwsgi.mule_id()
return instance_id
def start(self):
# Does a generalized `is_worker` attribute make sense? Hard to say w/o other stack paradigms.
if self._is_mule and self._farm_name:
# used by main.py to send a shutdown message on termination
os.environ['_GALAXY_UWSGI_FARM_NAMES'] = ','.join(self._farms)
super(UWSGIApplicationStack, self).start()
def has_pool(self, pool_name):
return pool_name in self._configured_farms
def in_pool(self, pool_name):
if not self._is_mule:
return False
else:
return pool_name in self._farms
def workers(self):
return uwsgi.workers()
def set_postfork_server_name(self, app):
app.config.server_name += ".%d" % uwsgi.worker_id()
@property
def facts(self):
facts = super(UWSGIApplicationStack, self).facts
if not self._is_mule:
facts.update({
'pool_name': 'web',
'server_id': uwsgi.worker_id(),
})
else:
facts.update({
'pool_name': self._farm_name,
'server_id': uwsgi.mule_id(),
})
facts['instance_id'] = self.instance_id
return facts
def shutdown(self):
super(UWSGIApplicationStack, self).shutdown()
class PasteApplicationStack(ApplicationStack):
@@ -91,16 +329,51 @@ def application_stack_class():
return WeblessApplicationStack
def application_stack_instance():
def application_stack_instance(app=None, config=None):
stack_class = application_stack_class()
return stack_class()
return stack_class(app=app, config=config)
def application_stack_log_filter():
return application_stack_class().log_filter_class()
def application_stack_log_formatter():
return logging.Formatter(fmt=application_stack_class().log_format)
def register_postfork_function(f, *args, **kwargs):
application_stack_class().register_postfork_function(f, *args, **kwargs)
@uwsgi_postfork
def _do_postfork():
def get_app_kwds(config_section, app_name=None):
return application_stack_class().get_app_kwds(config_section, app_name=app_name)
def get_stack_facts(config=None):
return application_stack_instance(config=config).facts
def _uwsgi_configured_mules():
mules = uwsgi.opt.get('mule', [])
return [mules] if isinstance(mules, string_types) or mules is True else mules
def _do_uwsgi_postfork():
for i, mule in enumerate(_uwsgi_configured_mules()):
if mule is not True and i + 1 == uwsgi.mule_id():
# mules will inherit the postfork function list and call them immediately upon fork, but programmed mules
# should not do that (they will call the postfork functions in-place as they start up after exec())
UWSGIApplicationStack.postfork_functions = [(_mule_fixup, (), {})]
for f, args, kwargs in [t for t in UWSGIApplicationStack.postfork_functions]:
log.debug('Calling postfork function: %s', f)
f(*args, **kwargs)
def _mule_fixup():
import urllib2
urllib2._opener = None
if uwsgi:
uwsgi.post_fork_hook = _do_uwsgi_postfork
lib/galaxy/web/stack/message.py
+168
View File
@@ -0,0 +1,168 @@
"""Web Application Stack worker messaging
"""
from __future__ import absolute_import
import json
import logging
from types import MethodType
log = logging.getLogger(__name__)
class ApplicationStackMessageDispatcher(object):
def __init__(self):
self.__funcs = {}
def __func_name(self, func, name):
if not name:
name = func.__name__
return name
def register_func(self, func, name=None):
name = self.__func_name(func, name)
self.__funcs[name] = func
def deregister_func(self, func=None, name=None):
name = self.__func_name(func, name)
try:
del self.__funcs[name]
except KeyError:
pass
@property
def handler_count(self):
return len(self.__funcs)
def dispatch(self, msg_str):
msg = decode(msg_str)
try:
msg.validate()
except AssertionError as exc:
log.error('Invalid message received: %s, error: %s', msg_str, exc)
return
if msg.target not in self.__funcs:
log.error("Received message with target '%s' but no functions were registered with that name. Params were: %s", msg.target, msg.params)
else:
self.__funcs[msg.target](msg)
class ApplicationStackMessage(dict):
target = None
default_handler = None
_validate_kwargs = ('target',)
def __init__(self, target=None, **kwargs):
self['target'] = target or self.__class__.target
self._merge_class_tuples()
def _merge_class_tuples(self):
"""Locates any class-level tuples beginning with a single (but not double) underscore in the MRO and creates a
property on the instance with the same name (without the leading underscore) that will return the union of
those tuples.
"""
names = set()
for cls in reversed(self.__class__.mro()):
names.update([x for x in dir(cls) if x.startswith('_') and not x.startswith('__') and type(getattr(cls, x)) == tuple])
for name in names:
setattr(self.__class__, name.lstrip('_'), property(lambda self, name=name: self._get_list_from_mro(name)))
def _get_list_from_mro(self, name):
"""Locate all class-level tuples with the given `name` in the MRO and return their union.
"""
r = set()
for cls in reversed(self.__class__.mro()):
r.update(getattr(cls, name, []))
return r
def _validate_items(self, obj, items, name):
for item in items:
assert item in obj, "Missing '%s' message %" % (item, name)
def validate(self):
self._validate_items(self, self.validate_kwargs, 'argument')
def encode(self):
self['__classname__'] = self.__class__.__name__
return json.dumps(self)
def bind_default_handler(self, obj, name):
"""Bind the default handler method to `obj` as attribute `name`.
This could also be implemented as a mixin class.
"""
assert self.default_handler is not None, '%s has no default handler method, cannot bind' % self.__class__.__name__
setattr(obj, name, MethodType(self.default_handler, obj, obj.__class__))
log.debug("Bound default message handler '%s.%s' to %s", self.__class__.__name__, self.default_handler.__name__,
getattr(obj, name))
@property
def target(self):
return self['target']
@target.setter
def set_target(self, target):
self['target'] = target
class ParamMessage(ApplicationStackMessage):
_validate_kwargs = ('params',)
_validate_params = ()
_exclude_params = ()
def __init__(self, target=None, params=None, **kwargs):
super(ParamMessage, self).__init__(target=target)
self['params'] = params or {}
for k, v in kwargs.items():
self['params'][k] = v
def validate(self):
super(ParamMessage, self).validate()
self._validate_items(self['params'], self.validate_params, 'parameters')
@property
def params(self):
d = self['params'].copy()
for key in self.exclude_params:
d.pop(key, None)
return d
@params.setter
def set_params(self, params):
self['params'] = params
class TaskMessage(ParamMessage):
_validate_params = ('task',)
_exclude_params = ('task',)
@staticmethod
def default_handler(self, msg):
"""Can be bound to an instance of any class that has message handling methods named like `_handle_{task}_method`
"""
name = '_handle_{task}_msg'.format(task=msg.task)
assert name in dir(self), "{cls} has no method _handle_{task}_msg, cannot handle message: {msg}".format(
cls=self.__class__.__name__,
task=msg.task,
msg=msg)
getattr(self, '_handle_%s_msg' % msg.task)(**msg.params)
@property
def task(self):
return self['params']['task']
class JobHandlerMessage(TaskMessage):
target = 'job_handler'
_validate_params = ('job_id',)
class WorkflowSchedulingMessage(TaskMessage):
target = 'workflow_scheduling'
_validate_params = ('workflow_invocation_id',)
def decode(msg_str):
d = json.loads(msg_str)
cls = d.pop('__classname__')
return globals()[cls](**d)
lib/galaxy/web/stack/transport.py
+152
View File
@@ -0,0 +1,152 @@
"""Web application stack operations
"""
from __future__ import absolute_import
import logging
import sys
import threading
try:
import uwsgi
except ImportError:
uwsgi = None
log = logging.getLogger(__name__)
class ApplicationStackTransport(object):
SHUTDOWN_MSG = '__SHUTDOWN__'
def __init__(self, app, stack, dispatcher=None):
""" Pre-fork initialization.
"""
self.app = app
self.stack = stack
self.can_run = False
self.running = False
self.dispatcher = dispatcher
self.dispatcher_thread = None
def _dispatch_messages(self):
pass
def start_if_needed(self):
# Don't unnecessarily start a thread that we don't need.
if self.can_run and not self.running and not self.dispatcher_thread and self.dispatcher and self.dispatcher.handler_count:
self.running = True
self.dispatcher_thread = threading.Thread(name=self.__class__.__name__ + ".dispatcher_thread", target=self._dispatch_messages)
self.dispatcher_thread.start()
log.info('%s dispatcher started', self.__class__.__name__)
def stop_if_unneeded(self):
if self.can_run and self.running and self.dispatcher_thread and self.dispatcher and not self.dispatcher.handler_count:
self.running = False
self.dispatcher_thread.join()
self.dispatcher_thread = None
log.info('%s dispatcher stopped', self.__class__.__name__)
def start(self):
""" Post-fork initialization.
"""
self.can_run = True
self.start_if_needed()
def send_message(self, msg, dest):
pass
def shutdown(self):
self.running = False
if self.dispatcher_thread:
log.info('Joining application stack transport dispatcher thread')
self.dispatcher_thread.join()
self.dispatcher_thread = None
class UWSGIFarmMessageTransport(ApplicationStackTransport):
""" Communication via uWSGI Mule Farm messages. Communication is unidirectional (workers -> mules).
"""
# Define any static lock names here, additional locks will be appended for each configured farm's message handler
_locks = []
def __initialize_locks(self):
num = int(uwsgi.opt.get('locks', 0)) + 1
farms = self.stack._configured_farms.keys()
need = len(farms)
if num < need:
raise RuntimeError('Need %i uWSGI locks but only %i exist(s): Set `locks = %i` in uWSGI configuration' % (need, num, need - 1))
sys.exit(1)
self._locks.extend(map(lambda x: 'RECV_MSG_FARM_' + x, farms))
# this would be nice, but in my 2.0.15 uWSGI, the uwsgi module has no set_option function, and I don't know if it'd work even if the function existed as documented
# if len(self.lock_map) > 1:
# uwsgi.set_option('locks', len(self.lock_map))
# log.debug('Created %s uWSGI locks' % len(self.lock_map))
def __init__(self, app, stack, dispatcher=None):
super(UWSGIFarmMessageTransport, self).__init__(app, stack, dispatcher=dispatcher)
self.__initialize_locks()
def __lock(self, name_or_id):
try:
uwsgi.lock(name_or_id)
except TypeError:
uwsgi.lock(self._locks.index(name_or_id))
def __unlock(self, name_or_id):
try:
uwsgi.unlock(name_or_id)
except TypeError:
uwsgi.unlock(self._locks.index(name_or_id))
def _farm_recv_msg_lock_num(self):
return self._locks.index('RECV_MSG_FARM_' + self.stack._farm_name)
def _dispatch_messages(self):
# this could be moved to the base class if locking was abstracted and a get_message method was added
log.info('Application stack message dispatcher thread starting up')
# we are going to do this a lot, so cache the lock number
lock = self._farm_recv_msg_lock_num()
while self.running:
msg = None
self.__lock(lock)
try:
log.debug('Acquired message lock, waiting for new message')
msg = uwsgi.farm_get_msg()
log.debug('Received message: %s', msg)
if msg == self.SHUTDOWN_MSG:
self.running = False
else:
self.dispatcher.dispatch(msg)
except Exception:
log.exception('Exception in mule message handling')
finally:
self.__unlock(lock)
log.debug('Released lock')
log.info('Application stack message dispatcher thread exiting')
# TODO: start_if_needed would be called on a web worker by the stack's register_message_handler function if a
# function were registered in a web handler, that should probably be prevented.
def start(self):
""" Post-fork initialization.
This is mainly done here for the future possibility that we'll be able to run mules post-fork without exec()ing. In a programmed mule it could be done at __init__ time.
"""
if self.stack._is_mule:
if not uwsgi.in_farm():
raise RuntimeError('Mule %s is not in a farm! Set `farm = <pool_name>:%s` in uWSGI configuration'
% (uwsgi.mule_id(),
','.join(map(str, range(1, len(filter(lambda x: x.endswith('galaxy/main.py'), self.stack._configured_mules)) + 1)))))
elif len(self.stack._farms) > 1:
raise RuntimeError('Mule %s is in multiple farms! This configuration is not supported due to locking issues' % uwsgi.mule_id())
# only mules receive messages so don't bother starting the dispatcher if we're not a mule (although
# currently it doesn't have any registered handlers and so wouldn't start anyway)
super(UWSGIFarmMessageTransport, self).start()
def shutdown(self):
if self.stack._is_mule:
super(UWSGIFarmMessageTransport, self).shutdown()
def send_message(self, msg, dest):
log.debug('Sending message to farm %s: %s', dest, msg)
uwsgi.farm_msg(dest, msg)
lib/galaxy/webapps/galaxy/api/histories.py
+9 -2
View File
@@ -317,9 +317,16 @@ class HistoriesController(BaseAPIController, ExportsHistoryMixin, ImportsHistory
if "archive_source" in payload:
archive_source = payload["archive_source"]
archive_type = payload.get("archive_type", "url")
archive_file = payload.get("archive_file")
if archive_source:
archive_type = payload.get("archive_type", "url")
elif hasattr(archive_file, "file"):
archive_source = payload["archive_file"].file.name
archive_type = "file"
else:
raise exceptions.MessageException("Please provide a url or file.")
self.queue_history_import(trans, archive_type=archive_type, archive_source=archive_source)
return {}
return {"message": "Importing history from source '%s'. This history will be visible when the import is complete." % archive_source}
new_history = None
# if a history id was passed, copy that history
lib/galaxy/webapps/galaxy/buildapp.py
+17 -31
View File
@@ -3,13 +3,11 @@ Provides factory methods to assemble the Galaxy web application
"""
import atexit
import logging
import os
import sys
import threading
import traceback
from paste import httpexceptions
from six.moves import configparser
import galaxy.app
import galaxy.datatypes.registry
@@ -34,16 +32,13 @@ class GalaxyWebApplication(galaxy.web.framework.webapp.WebApplication):
pass
def app_factory(global_conf, **kwargs):
return paste_app_factory(global_conf, **kwargs)
def paste_app_factory(global_conf, **kwargs):
def app_factory(global_conf, load_app_kwds={}, **kwargs):
"""
Return a wsgi application serving the root object
"""
kwargs = load_app_properties(
kwds=kwargs
kwds=kwargs,
**load_app_kwds
)
# Create the Galaxy application unless passed in
if 'app' in kwargs:
@@ -58,7 +53,7 @@ def paste_app_factory(global_conf, **kwargs):
sys.exit(1)
# Call app's shutdown method when the interpeter exits, this cleanly stops
# the various Galaxy application daemon threads
atexit.register(app.shutdown)
app.application_stack.register_postfork_function(atexit.register, app.shutdown)
# Create the universe WSGI application
webapp = GalaxyWebApplication(app, session_cookie='galaxysession', name='galaxy')
@@ -105,14 +100,12 @@ def paste_app_factory(global_conf, **kwargs):
webapp.add_client_route('/admin/repositories', 'admin')
webapp.add_client_route('/admin/tool_versions', 'admin')
webapp.add_client_route('/admin/quotas', 'admin')
webapp.add_client_route('/admin/forms/{form_id}', 'admin')
webapp.add_client_route('/admin/form/{form_id}', 'admin')
webapp.add_client_route('/admin/api_keys', 'admin')
webapp.add_client_route('/tours')
webapp.add_client_route('/tours/{tour_id}')
webapp.add_client_route('/user')
webapp.add_client_route('/user/{form_id}')
webapp.add_client_route('/workflow')
webapp.add_client_route('/workflows/list_published')
webapp.add_client_route('/visualizations/list_published')
webapp.add_client_route('/visualizations/list')
webapp.add_client_route('/visualizations/edit')
@@ -122,6 +115,7 @@ def paste_app_factory(global_conf, **kwargs):
webapp.add_client_route('/pages/edit')
webapp.add_client_route('/histories/citations')
webapp.add_client_route('/histories/list')
webapp.add_client_route('/histories/import')
webapp.add_client_route('/histories/list_published')
webapp.add_client_route('/histories/list_shared')
webapp.add_client_route('/histories/rename')
@@ -129,8 +123,11 @@ def paste_app_factory(global_conf, **kwargs):
webapp.add_client_route('/datasets/list')
webapp.add_client_route('/datasets/edit')
webapp.add_client_route('/datasets/error')
webapp.add_client_route('/workflow/run')
webapp.add_client_route('/workflow/import_workflow')
webapp.add_client_route('/workflows/list')
webapp.add_client_route('/workflows/list_published')
webapp.add_client_route('/workflows/create')
webapp.add_client_route('/workflows/run')
webapp.add_client_route('/workflows/import_workflow')
webapp.add_client_route('/custom_builds')
# ==== Done
@@ -165,27 +162,16 @@ def paste_app_factory(global_conf, **kwargs):
return webapp
def uwsgi_app_factory():
# TODO: synchronize with galaxy.web.framework.webapp.build_native_uwsgi_app - should
# at least be using nice_config_parser for instance.
import uwsgi
root = os.path.abspath(uwsgi.opt.get('galaxy_root', os.getcwd()))
config_file = uwsgi.opt.get('galaxy_config_file', os.path.join(root, 'config', 'galaxy.ini'))
global_conf = {
'__file__': config_file if os.path.exists(__file__) else None,
'here': root}
parser = configparser.ConfigParser()
parser.read(config_file)
try:
kwargs = dict(parser.items('app:main'))
except configparser.NoSectionError:
kwargs = {}
return app_factory(global_conf, **kwargs)
def uwsgi_app():
return galaxy.web.framework.webapp.build_native_uwsgi_app(app_factory, "galaxy")
# For backwards compatibility
uwsgi_app_factory = uwsgi_app
def postfork_setup():
from galaxy.app import app
app.application_stack.set_postfork_server_name(app)
app.control_worker.bind_and_start()
lib/galaxy/webapps/galaxy/controllers/dataset.py
+1 -1
View File
@@ -309,7 +309,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
attribute_inputs.append({
'type' : 'select',
'multiple' : attributes.multiple,
'optional' : attributes.optional,
'optional' : spec.get('optional'),
'name' : name,
'label' : spec.desc,
'options' : attributes.options,
lib/galaxy/webapps/galaxy/controllers/history.py
-27
View File
@@ -1185,33 +1185,6 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
return self.get_ave_item_rating_data(trans.sa_session, history)
# TODO: used in display_base.mako
@web.expose
# TODO: Remove require_login when users are warned that, if they are not
# logged in, this will remove their current history.
@web.require_login("use Galaxy histories")
def import_archive(self, trans, **kwargs):
""" Import a history from a file archive. """
# Set archive source and type.
archive_file = kwargs.get('archive_file', None)
archive_url = kwargs.get('archive_url', None)
archive_source = None
if hasattr(archive_file, 'file'):
archive_source = archive_file.file.name
archive_type = 'file'
elif archive_url:
archive_source = archive_url
archive_type = 'url'
# If no source to create archive from, show form to upload archive or specify URL.
if not archive_source:
form = web.FormBuilder(web.url_for(controller='history', action='import_archive'), "Import a History from an Archive", submit_text="Submit")
form.add_input("text", "Archived History URL", "archive_url", value="", error=None)
form.add_input("file", "Archived History File", "archive_file", value="", error=None)
return trans.show_form(form)
self.queue_history_import(trans, archive_type=archive_type, archive_source=archive_source)
return trans.show_message("Importing history from '%s'. \
This history will be visible when the import is complete" % archive_source)
# TODO: used in this file and index.mako
@web.expose
def export_archive(self, trans, id=None, gzip=True, include_hidden=False, include_deleted=False, preview=False):
""" Export a history to an archive. """
lib/galaxy/webapps/galaxy/controllers/workflow.py
+24 -42
View File
@@ -33,11 +33,9 @@ from galaxy.web.base.controller import (
SharableMixin,
UsesStoredWorkflowMixin
)
from galaxy.web.framework.formbuilder import form
from galaxy.web.framework.helpers import (
grids,
time_ago,
to_unicode
)
from galaxy.workflow.extract import (
extract_workflow,
@@ -385,28 +383,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
# Redirect to load galaxy frames.
return trans.show_ok_message(
message="""Workflow "%s" has been imported. <br>You can <a href="%s">start using this workflow</a> or %s."""
% (stored.name, web.url_for(controller='workflow'), referer_message), use_panels=True)
@web.expose
@web.require_login("use Galaxy workflows")
def rename(self, trans, id, new_name=None, **kwargs):
stored = self.get_stored_workflow(trans, id)
if new_name is not None:
san_new_name = sanitize_html(new_name)
stored.name = san_new_name
stored.latest_workflow.name = san_new_name
trans.sa_session.flush()
message = 'Workflow renamed to: %s' % escape(san_new_name)
trans.set_message(message)
# Take care of proxy prefix in url as well
redirect_url = url_for('/') + 'workflow?status=done&message=%s' % escape(message)
return trans.response.send_redirect(redirect_url)
else:
return form(url_for(controller='workflow', action='rename', id=trans.security.encode_id(stored.id)),
"Rename workflow",
submit_text="Rename",
use_panels=True) \
.add_text("new_name", "Workflow Name", value=to_unicode(stored.name))
% (stored.name, web.url_for('/workflows/list'), referer_message))
@web.expose
@web.require_login("use Galaxy workflows")
@@ -537,14 +514,26 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
return_url = url_for('/') + 'workflow?status=done&message=%s' % escape(message)
trans.response.send_redirect(return_url)
@web.expose
@web.require_login("create workflows")
def create(self, trans, workflow_name=None, workflow_annotation=""):
"""
Create a new stored workflow with name `workflow_name`.
"""
user = trans.get_user()
if workflow_name is not None:
@web.expose_api
def create(self, trans, payload=None, **kwd):
if trans.request.method == 'GET':
return {
'title' : 'Create Workflow',
'inputs' : [{
'name' : 'workflow_name',
'label' : 'Name',
'value' : 'Unnamed workflow'
}, {
'name' : 'workflow_annotation',
'label' : 'Annotation',
'help' : 'A description of the workflow; annotation is shown alongside shared or published workflows.'
}]}
else:
user = trans.get_user()
workflow_name = payload.get('workflow_name')
workflow_annotation = payload.get('workflow_annotation')
if not workflow_name:
return self.message_exception(trans, 'Please provide a workflow name.')
# Create the new stored workflow
stored_workflow = model.StoredWorkflow()
stored_workflow.name = workflow_name
@@ -562,14 +551,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
session = trans.sa_session
session.add(stored_workflow)
session.flush()
return self.editor(trans, id=trans.security.encode_id(stored_workflow.id))
else:
return form(url_for(controller="workflow", action="create"), "Create New Workflow", submit_text="Create", use_panels=True) \
.add_text("workflow_name", "Workflow Name", value="Unnamed workflow") \
.add_text("workflow_annotation",
"Workflow Annotation",
value="",
help="A description of the workflow; annotation is shown alongside shared or published workflows.")
return {'id': trans.security.encode_id(stored_workflow.id), 'message': 'Workflow %s has been created.' % workflow_name}
@web.json
def save_workflow_as(self, trans, workflow_name, workflow_data, workflow_annotation=""):
@@ -922,12 +904,12 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
id=repository_id,
message=message,
status=status))
redirect_url = url_for('/') + 'workflow?status=' + status + '&message=%s' % escape(message)
redirect_url = url_for('/') + 'workflows/list?status=' + status + '&message=%s' % escape(message)
return trans.response.send_redirect(redirect_url)
if cntrller == 'api':
return status, message
if status == 'error':
redirect_url = url_for('/') + 'workflow?status=' + status + '&message=%s' % escape(message)
redirect_url = url_for('/') + 'workflows/list?status=' + status + '&message=%s' % escape(message)
return trans.response.send_redirect(redirect_url)
else:
return {
lib/galaxy/webapps/tool_shed/config.py
-7
View File
@@ -105,7 +105,6 @@ class Configuration(object):
self.smtp_username = kwargs.get('smtp_username', None)
self.smtp_password = kwargs.get('smtp_password', None)
self.smtp_ssl = kwargs.get('smtp_ssl', None)
self.start_job_runners = kwargs.get('start_job_runners', None)
self.email_from = kwargs.get('email_from', None)
self.nginx_upload_path = kwargs.get('nginx_upload_path', False)
self.log_actions = string_as_bool(kwargs.get('log_actions', 'False'))
@@ -123,12 +122,6 @@ class Configuration(object):
self.log_events = False
self.cloud_controller_instance = False
self.server_name = ''
self.job_manager = ''
self.default_job_handlers = []
self.default_cluster_job_runner = 'local:///'
self.job_handlers = []
self.tool_handlers = []
self.tool_runners = []
# Error logging with sentry
self.sentry_dsn = kwargs.get('sentry_dsn', None)
# Where the tool shed hgweb.config file is stored - the default is the Galaxy installation directory.
lib/galaxy/workflow/scheduling_manager.py
+54 -6
View File
@@ -7,6 +7,7 @@ from galaxy import model
from galaxy.util import plugin_config
from galaxy.util.handlers import ConfiguresHandlers
from galaxy.util.monitors import Monitors
from galaxy.web.stack.message import WorkflowSchedulingMessage
log = logging.getLogger(__name__)
@@ -30,11 +31,16 @@ class WorkflowSchedulingManager(object, ConfiguresHandlers):
self.__handlers_configured = False
self.workflow_schedulers = {}
self.active_workflow_schedulers = {}
# TODO: this should not hardcode the job handlers pool
self.__handler_pool = self.app.application_stack.pools.JOB_HANDLERS
# TODO: and we need a better way to indicate messaging should be used
self.__use_stack_messages = app.application_stack.has_pool(self.__handler_pool)
# Passive workflow schedulers won't need to be monitored I guess.
self.request_monitor = None
self.handlers = {}
self._is_handler = None
self.__plugin_classes = self.__plugins_dict()
self.__init_schedulers()
@@ -44,9 +50,14 @@ class WorkflowSchedulingManager(object, ConfiguresHandlers):
self.__start_schedulers()
if self.active_workflow_schedulers:
self.__start_request_monitor()
if self.__use_stack_messages:
WorkflowSchedulingMessage().bind_default_handler(self, '_handle_message')
self.app.application_stack.register_message_handler(
self._handle_message,
name=WorkflowSchedulingMessage.target)
else:
# Process should not schedule workflows - do nothing.
pass
# Process should not schedule workflows but should check for any unassigned to handlers
self.__startup_recovery()
# When assinging handlers to workflows being queued - use job_conf
# if not explicit workflow scheduling handlers have be specified or
@@ -56,13 +67,36 @@ class WorkflowSchedulingManager(object, ConfiguresHandlers):
else:
self.__has_handlers = app.job_config
def __startup_recovery(self):
sa_session = self.app.model.context
if self.__use_stack_messages:
for workflow_invocation in model.WorkflowInvocation.poll_active_workflow_ids(
sa_session,
handler=None):
log.info("(%s) Handler unassigned at startup, queueing workflow invocation via stack messaging for pool"
" [%s]", workflow_invocation.id, self.__handler_pool)
msg = WorkflowSchedulingMessage(task='setup', workflow_invocation_id=workflow_invocation.id)
self.app.application_stack.send_message(self.app.application_stack.pools.JOB_HANDLERS, msg)
def _handle_setup_msg(self, workflow_invocation_id=None):
sa_session = self.app.model.context
workflow_invocation = sa_session.query(model.WorkflowInvocation).get(workflow_invocation_id)
if workflow_invocation.handler is None:
workflow_invocation.handler = self.app.config.server_name
sa_session.add(workflow_invocation)
sa_session.flush()
else:
log.warning("(%s) Handler '%s' received setup message for workflow invocation but handler '%s' is"
" already assigned, ignoring", workflow_invocation.id, self.app.config.server_name,
workflow_invocation.handler)
def _is_workflow_handler(self):
# If we have explicitly configured handlers, check them.
# Else just make sure we are a job handler.
if self.__handlers_configured:
is_handler = self.is_handler(self.app.config.server_name)
is_handler = self.is_handler
else:
is_handler = self.app.is_job_handler()
is_handler = self.app.is_job_handler
return is_handler
def _get_handler(self, history_id):
@@ -95,14 +129,24 @@ class WorkflowSchedulingManager(object, ConfiguresHandlers):
workflow_invocation.state = model.WorkflowInvocation.states.NEW
scheduler = request_params.get("scheduler", None) or self.default_scheduler_id
handler = self._get_handler(workflow_invocation.history.id)
log.info("Queueing workflow invocation for handler [%s]" % handler)
if handler is None and not self.__use_stack_messages:
raise RuntimeError("Unable to set a handler for workflow invocation '%s'" % workflow_invocation.id)
log.info("Queueing workflow invocation for handler [%s]", handler)
workflow_invocation.scheduler = scheduler
workflow_invocation.handler = handler
sa_session = self.app.model.context
sa_session.add(workflow_invocation)
sa_session.flush()
if handler is None and self.__use_stack_messages:
log.info("(%s) Queueing workflow invocation via stack messaging for pool [%s]",
workflow_invocation.id, self.__handler_pool)
msg = WorkflowSchedulingMessage(task='setup', workflow_invocation_id=workflow_invocation.id)
self.app.application_stack.send_message(self.__handler_pool, msg)
return workflow_invocation
def __start_schedulers(self):
@@ -173,6 +217,7 @@ class WorkflowSchedulingManager(object, ConfiguresHandlers):
def __start_request_monitor(self):
self.request_monitor = WorkflowRequestMonitor(self.app, self)
self.app.application_stack.register_postfork_function(self.request_monitor.start)
class WorkflowRequestMonitor(Monitors, object):
@@ -180,7 +225,7 @@ class WorkflowRequestMonitor(Monitors, object):
def __init__(self, app, workflow_scheduling_manager):
self.app = app
self.workflow_scheduling_manager = workflow_scheduling_manager
self._init_monitor_thread(name="WorkflowRequestMonitor.monitor_thread", target=self.__monitor, start=True, config=app.config)
self._init_monitor_thread(name="WorkflowRequestMonitor.monitor_thread", target=self.__monitor, config=app.config)
def __monitor(self):
to_monitor = self.workflow_scheduling_manager.active_workflow_schedulers
@@ -235,5 +280,8 @@ class WorkflowRequestMonitor(Monitors, object):
handler=handler,
)
def start(self):
self.monitor_thread.start()
def shutdown(self):
self.shutdown_monitor()
lib/tool_shed/galaxy_install/migrate/check.py
+3 -4
View File
@@ -55,11 +55,10 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options={}):
if not app.config.running_functional_tests:
if tool_shed_accessible:
# Automatically update the value of the migrate_tools.version database table column.
config_arg = ''
cmd = ['sh', 'manage_db.sh', 'upgrade', 'tools']
if galaxy_config_file:
config_arg = " -c %s" % galaxy_config_file
cmd = 'sh manage_tools.sh%s upgrade' % config_arg
proc = subprocess.Popen(args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
cmd[2:2] = ['-c', galaxy_config_file]
proc = subprocess.Popen(args=cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
return_code = proc.wait()
output = proc.stdout.read(32768)
if return_code != 0:
lib/tool_shed/util/commit_util.py
+2 -1
View File
@@ -10,7 +10,8 @@ from collections import namedtuple
from sqlalchemy.sql.expression import null
import tool_shed.repository_types.util as rt_util
from galaxy.util import checkers, safe_relpath
from galaxy.util import checkers
from galaxy.util.path import safe_relpath
from tool_shed.tools import data_table_manager
from tool_shed.util import basic_util, hg_util, shed_util_common as suc
lib/tool_shed/util/tool_util.py
+1 -35
View File
@@ -4,8 +4,8 @@ import shutil
import galaxy.tools
from galaxy import util
from galaxy.datatypes.sniff import is_column_based
from galaxy.util import checkers
from galaxy.util import unicodify
from galaxy.util.expressions import ExpressionContext
from galaxy.web.form_builder import SelectField
from tool_shed.util import basic_util
@@ -125,20 +125,6 @@ def generate_message_for_invalid_tools(app, invalid_file_tups, repository, metad
return message
def get_headers(fname, sep, count=60, is_multi_byte=False):
"""Returns a list with the first 'count' lines split by 'sep'."""
headers = []
for idx, line in enumerate(open(fname)):
line = line.rstrip('\n\r')
if is_multi_byte:
line = unicodify(line, 'utf-8')
sep = sep.encode('utf-8')
headers.append(line.split(sep))
if idx == count:
break
return headers
def get_tool_path_install_dir(partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems):
for elem in config_elems:
if elem.tag == 'tool':
@@ -184,26 +170,6 @@ def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tup
return repository_tools_tups, sample_files_copied
def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False):
"""See if the file is column based with respect to a separator."""
headers = get_headers(fname, sep, is_multi_byte=is_multi_byte)
count = 0
if not headers:
return False
for hdr in headers[skip:]:
if hdr and hdr[0] and not hdr[0].startswith('#'):
if len(hdr) > 1:
count = len(hdr)
break
if count < 2:
return False
for hdr in headers[skip:]:
if hdr and hdr[0] and not hdr[0].startswith('#'):
if len(hdr) != count:
return False
return True
def is_data_index_sample_file(file_path):
"""
Attempt to determine if a .sample file is appropriate for copying to ~/tool-data when
manage_tools.sh
-12
View File
@@ -1,12 +0,0 @@
#!/bin/sh
cd `dirname $0`
: ${GALAXY_VIRTUAL_ENV:=.venv}
if [ -d "$GALAXY_VIRTUAL_ENV" ];
then
printf "Activating virtualenv at $GALAXY_VIRTUAL_ENV\n"
. "$GALAXY_VIRTUAL_ENV/bin/activate"
fi
python ./scripts/manage_tools.py $@
run.sh
+29 -7
View File
@@ -1,5 +1,12 @@
#!/bin/sh
# Usage: ./run.sh <start|stop|restart>
#
#
# Description: This script can be used to start or stop the galaxy
# web application.
cd "$(dirname "$0")"
. ./scripts/common_startup_functions.sh
@@ -16,6 +23,11 @@ then
. $GALAXY_LOCAL_ENV_FILE
fi
GALAXY_PID=${GALAXY_PID:-galaxy.pid}
GALAXY_LOG=${GALAXY_LOG:-galaxy.log}
PID_FILE=$GALAXY_PID
LOG_FILE=$GALAXY_LOG
parse_common_args $@
run_common_start_up
@@ -39,18 +51,27 @@ if [ -z "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=universe_wsgi.ini
elif [ -f config/galaxy.ini ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini
else
elif [ -f config/galaxy.yml ]; then
GALAXY_CONFIG_FILE=config/galaxy.yml
elif [ -f config/galaxy.ini.sample -a -z "$GALAXY_UWSGI" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini.sample
fi
export GALAXY_CONFIG_FILE
fi
if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then
# Install Conda environment if needed.
python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed
if [ -n "$GALAXY_CONFIG_FILE" ]; then
config_file_arg="-c $GALAXY_CONFIG_FILE"
fi
if [ -n "$GALAXY_RUN_ALL" ]; then
if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then
# Install Conda environment if needed.
python ./scripts/manage_tool_dependencies.py $config_file_arg init_if_needed
fi
[ -n "$GALAXY_UWSGI" ] && APP_WEBSERVER='uwsgi'
find_server ${GALAXY_CONFIG_FILE:-none} galaxy
if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then
servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo)
if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then
echo "ERROR: \$GALAXY_RUN_ALL cannot be used without the '--daemon', '--stop-daemon' or 'restart' arguments to run.sh"
@@ -80,6 +101,7 @@ if [ -n "$GALAXY_RUN_ALL" ]; then
fi
done
else
# Handle only 1 server, whose name can be specified with --server-name parameter (defaults to "main")
python ./scripts/paster.py serve "$GALAXY_CONFIG_FILE" $paster_args
echo "executing: $run_server $server_args"
# args are properly quoted so use eval
eval $run_server $server_args
fi
run_reports.sh
+3 -4
View File
@@ -39,8 +39,6 @@ if [ -z "$GALAXY_REPORTS_CONFIG" ]; then
GALAXY_REPORTS_CONFIG=config/reports.ini
elif [ -f config/reports.yml ]; then
GALAXY_REPORTS_CONFIG=config/reports.yml
else
GALAXY_REPORTS_CONFIG=config/reports.yml.sample
fi
export GALAXY_REPORTS_CONFIG
fi
@@ -49,5 +47,6 @@ if [ -n "$GALAXY_REPORTS_CONFIG_DIR" ]; then
python ./scripts/build_universe_config.py "$GALAXY_REPORTS_CONFIG_DIR" "$GALAXY_REPORTS_CONFIG"
fi
find_server $GALAXY_REPORTS_CONFIG
$run_server $server_args
find_server ${GALAXY_REPORTS_CONFIG:-none} reports
echo "executing: $run_server $server_args"
eval $run_server $server_args
run_tool_shed.sh
+3 -4
View File
@@ -32,11 +32,10 @@ if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then
TOOL_SHED_CONFIG_FILE=config/tool_shed.ini
elif [ -f config/tool_shed.yml ]; then
TOOL_SHED_CONFIG_FILE=config/tool_shed.yml
else
TOOL_SHED_CONFIG_FILE=config/tool_shed.yml.sample
fi
export TOOL_SHED_CONFIG_FILE
fi
find_server $TOOL_SHED_CONFIG_FILE
$run_server $server_args
find_server ${TOOL_SHED_CONFIG_FILE:-none} tool_shed
echo "executing: $run_server $server_args"
eval $run_server $server_args
scripts/common_startup_functions.sh
+12 -26
View File
@@ -1,9 +1,5 @@
#!/bin/sh
: ${GALAXY_STATIC_DIRECTORY:=static}
: ${GALAXY_STYLE_DIRECTORY:=static/style/blue}
uwsgi_args="--master --pythonpath=lib --static-map /static=$GALAXY_STATIC_DIRECTORY --static-map /static/style=$GALAXY_STYLE_DIRECTORY"
parse_common_args() {
INITIALIZE_TOOL_DEPENDENCIES=1
# Pop args meant for common_startup.sh
@@ -23,9 +19,9 @@ parse_common_args() {
common_startup_args="$common_startup_args $1"
shift
;;
--stop-daemon)
common_startup_args="$common_startup_args $1"
paster_args="$paster_args $1"
--stop-daemon|stop)
common_startup_args="$common_startup_args --stop-daemon"
paster_args="$paster_args --pid-file $PID_FILE --stop-daemon"
uwsgi_args="$uwsgi_args --stop $PID_FILE"
stop_daemon_arg_set=1
shift
@@ -41,8 +37,8 @@ parse_common_args() {
daemon_or_restart_arg_set=1
shift
;;
--daemon)
paster_args="$paster_args $1"
--daemon|start)
paster_args="$paster_args --pid-file $PID_FILE --log-file $LOG_FILE --daemon"
# --daemonize2 waits until after the application has loaded
# to daemonize, thus it stops if any errors are found
uwsgi_args="$uwsgi_args --daemonize2 $LOG_FILE --safe-pidfile $PID_FILE"
@@ -87,25 +83,13 @@ setup_python() {
python ./scripts/check_python.py || exit 1
}
find_uwsgi() {
# Look for uwsgi
if [ -z "$skip_venv" -a -x $GALAXY_VIRTUAL_ENV/bin/uwsgi ]; then
UWSGI=$GALAXY_VIRTUAL_ENV/bin/uwsgi
elif command -v uwsgi >/dev/null 2>&1; then
UWSGI=uwsgi
else
echo 'ERROR: Could not find uwsgi executable'
exit 1
fi
}
find_server() {
server_config="$1"
server_config_style="ini-paste"
server_app="$2"
arg_getter_args=
default_webserver="paste"
case "$server_config" in
*.y*ml*)
server_config_style="yaml"
*.y*ml|''|none)
default_webserver="uwsgi" # paste incapable of this
;;
esac
@@ -122,10 +106,12 @@ find_server() {
echo 'ERROR: Could not find uwsgi executable'
exit 1
fi
[ "$server_config" != "none" ] && arg_getter_args="-c $server_config"
[ -n "$server_app" ] && arg_getter_args="--app $server_app"
run_server="$UWSGI"
server_args="--$server_config_style $server_config $uwsgi_args"
server_args="$(python ./scripts/get_uwsgi_args.py $arg_getter_args) $uwsgi_args"
else
run_server="python"
server_args="./scripts/paster.py serve $server_config $paster_args --pid-file $PID_FILE --log-file $LOG_FILE $paster_args"
server_args="./scripts/paster.py serve $server_config $paster_args"
fi
}
scripts/galaxy-main
+101 -66
View File
@@ -15,12 +15,20 @@ a loggers section in Galaxy's ini file - this can be overridden with sensible
defaults logging to a single file with the following:
galaxy-main -d --server-name handler0 --daemon-log-file=handler0-daemon.log --pid-file handler0.pid --log-file handler0.log
This can also be used to start Galaxy as a uWSGI mule, e.g. for job handling:
uwsgi ... --py-call-osafterfork --mule=lib/galaxy/main.py --mule=lib/galaxy/main.py --farm=job-handlers:1,2
The --py-call-osafterfork allows for proper shutdown on SIGTERM/SIGINT.
"""
import functools
import logging
import os
import signal
import sys
import time
import threading
from argparse import ArgumentParser
from logging.config import fileConfig
try:
@@ -33,25 +41,10 @@ try:
except ImportError:
Daemonize = None
# Vaguely Python 2.6 compatibile ArgumentParser import
try:
from argparse import ArgumentParser
import uwsgi
except ImportError:
from optparse import OptionParser
class ArgumentParser(OptionParser):
def __init__(self, **kwargs):
self.delegate = OptionParser(**kwargs)
def add_argument(self, *args, **kwargs):
if "required" in kwargs:
del kwargs["required"]
return self.delegate.add_option(*args, **kwargs)
def parse_args(self, args=None):
(options, args) = self.delegate.parse_args(args)
return options
uwsgi = None
REQUIRES_DAEMONIZE_MESSAGE = "Attempted to use Galaxy in daemon mode, but daemonize is unavailable."
@@ -61,12 +54,19 @@ real_file = os.path.realpath(__file__)
GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir))
GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR, "lib")
DEFAULT_INI_APP = "main"
DEFAULT_CONFIG_SECTION = "galaxy"
DEFAULT_INIS = ["config/galaxy.ini", "universe_wsgi.ini", "config/galaxy.ini.sample"]
DEFAULT_PID = "galaxy.pid"
DEFAULT_VERBOSE = True
DESCRIPTION = "Daemonized entry point for Galaxy."
SHUTDOWN_MSG = '__SHUTDOWN__'
UWSGI_FARMS_VAR = '_GALAXY_UWSGI_FARM_NAMES'
exit = threading.Event()
def load_galaxy_app(
config_builder,
@@ -86,11 +86,6 @@ def load_galaxy_app(
except Exception:
log.exception("Failed to chdir")
raise
try:
sys.path.insert(1, GALAXY_LIB_DIR)
except Exception:
log.exception("Failed to add Galaxy to sys.path")
raise
config_builder.setup_logging()
from galaxy.util.properties import load_app_properties
@@ -98,13 +93,31 @@ def load_galaxy_app(
kwds = load_app_properties(**kwds)
from galaxy.app import UniverseApplication
app = UniverseApplication(
global_conf={"__file__": config_builder.ini_path},
global_conf=config_builder.global_conf(),
**kwds
)
app.control_worker.bind_and_start()
return app
def handle_signal(signum, frame):
log.info('Received signal %d, exiting', signum)
if uwsgi and 'mule_id' in dir(uwsgi) and uwsgi.mule_id() > 0:
farms = os.environ.get(UWSGI_FARMS_VAR, None)
if farms:
for farm in farms.split(','):
uwsgi.farm_msg(farm, SHUTDOWN_MSG)
else:
uwsgi.mule_msg(SHUTDOWN_MSG, uwsgi.mule_id())
exit.set()
def register_signals():
for name in ('TERM', 'INT', 'HUP'):
sig = getattr(signal, 'SIG%s' % name)
signal.signal(sig, handle_signal)
def app_loop(args, log):
try:
config_builder = GalaxyConfigBuilder(args)
@@ -116,16 +129,12 @@ def app_loop(args, log):
except BaseException:
log.exception("Failed to initialize Galaxy application")
raise
sleep = True
while sleep:
try:
time.sleep(5)
except KeyboardInterrupt:
sleep = False
except SystemExit:
sleep = False
except Exception:
try:
# A timeout is required or the signals won't be handled
while not exit.wait(20):
pass
except (KeyboardInterrupt, SystemExit):
pass
try:
galaxy_app.shutdown()
except Exception:
@@ -139,16 +148,16 @@ def absolute_config_path(path, galaxy_root):
return path
def find_ini(supplied_ini, galaxy_root):
if supplied_ini:
return supplied_ini
def find_config(supplied_config, galaxy_root):
if supplied_config:
return supplied_config
# If not explicitly supplied an ini, check server.ini and then
# just restort to sample if that has not been configured.
# If not explicitly supplied an config, check galaxy.ini and then
# just resort to sample if that has not been configured.
for guess in DEFAULT_INIS:
ini_path = os.path.join(galaxy_root, guess)
if os.path.exists(ini_path):
return ini_path
config_path = os.path.join(galaxy_root, guess)
if os.path.exists(config_path):
return config_path
return guess
@@ -158,54 +167,79 @@ class GalaxyConfigBuilder(object):
"""
def __init__(self, args=None, **kwds):
ini_path = kwds.get("ini_path", None) or (args and args.ini_path)
# If given app_conf_path - use that - else we need to ensure we have an
# ini path.
if not ini_path:
self.app_name = kwds.get("app") or (args and args.app) or DEFAULT_CONFIG_SECTION
config_file = kwds.get("config_file", None) or (args and args.config_file)
# If given app_conf_path - use that - else we need to ensure we have a
# config file path.
if not config_file and 'config_file' in self.app_kwds():
config_file = self.app_kwds()['config_file']
if not config_file:
galaxy_root = kwds.get("galaxy_root", GALAXY_ROOT_DIR)
ini_path = find_ini(ini_path, galaxy_root)
ini_path = absolute_config_path(ini_path, galaxy_root=galaxy_root)
self.ini_path = ini_path
self.app_name = kwds.get("app") or (args and args.app) or DEFAULT_INI_APP
config_file = find_config(config_file, galaxy_root)
config_file = absolute_config_path(config_file, galaxy_root=galaxy_root)
self.config_file = config_file
# FIXME: this won't work for non-Paste ini configs
self.config_section = self.app_name if not self.config_is_ini else 'app:%s' % (kwds.get("app") or (args and args.app) or DEFAULT_INI_APP)
self.log_file = (args and args.log_file)
@classmethod
def populate_options(cls, arg_parser):
arg_parser.add_argument("-c", "--ini-path", default=None, help="Galaxy ini config file (defaults to config/galaxy.ini)")
arg_parser.add_argument("--app", default=DEFAULT_INI_APP, help="app section in ini file (defaults to main)")
arg_parser.add_argument("-c", "--config-file", default=None, help="Galaxy config file (defaults to config/galaxy.ini)")
arg_parser.add_argument("--ini-path", default=None, help="DEPRECATED: use -c/--config-file")
arg_parser.add_argument("--app", default=None, help="app section in ini file (defaults to 'galaxy' for YAML/JSON, 'main' (w/ 'app:' prepended) for INI")
arg_parser.add_argument("-d", "--daemonize", default=False, help="Daemonzie process", action="store_true")
arg_parser.add_argument("--daemon-log-file", default=None, help="log file for daemon script ")
arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in ini_path if set)")
arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in config_file if set)")
arg_parser.add_argument("--pid-file", default=DEFAULT_PID, help="pid file (default is %s)" % DEFAULT_PID)
arg_parser.add_argument("--server-name", default=None, help="set a galaxy server name")
@property
def config_is_ini(self):
return self.config_file.endswith('.ini') or self.config_file.endswith('.ini.sample')
def app_kwds(self):
config = dict(
ini_file=self.ini_path,
ini_section="app:%s" % self.app_name,
)
return config
from galaxy.web.stack import get_app_kwds
kwds = get_app_kwds(self.app_name, app_name=self.app_name)
if 'config_file' not in kwds:
kwds['config_file'] = self.config_file
if 'config_section' not in kwds:
kwds['config_section'] = self.config_section
return kwds
def global_conf(self):
conf = {}
if self.config_is_ini:
conf["__file__"] = self.config_file
return conf
def setup_logging(self):
# Galaxy will attempt to setup logging if loggers is not present in
# ini config file - this handles that loggers block however if present
# (the way paste normally would)
if not self.ini_path:
if not self.config_file:
return
raw_config = configparser.ConfigParser()
raw_config.read([self.ini_path])
if raw_config.has_section('loggers'):
config_file = os.path.abspath(self.ini_path)
fileConfig(
config_file,
dict(__file__=config_file, here=os.path.dirname(config_file))
)
if self.config_is_ini:
raw_config = configparser.ConfigParser()
raw_config.read([self.config_file])
if raw_config.has_section('loggers'):
config_file = os.path.abspath(self.config_file)
fileConfig(
config_file,
dict(__file__=config_file, here=os.path.dirname(config_file))
)
def main():
arg_parser = ArgumentParser(description=DESCRIPTION)
try:
sys.path.insert(1, GALAXY_LIB_DIR)
except Exception:
log.exception("Failed to add Galaxy to sys.path")
raise
GalaxyConfigBuilder.populate_options(arg_parser)
args = arg_parser.parse_args()
if args.ini_path and not args.config_file:
args.config_file = args.ini_path
if args.log_file:
os.environ["GALAXY_CONFIG_LOG_DESTINATION"] = os.path.abspath(args.log_file)
if args.server_name:
@@ -214,6 +248,7 @@ def main():
log.setLevel(logging.DEBUG)
log.propagate = False
register_signals()
if args.daemonize:
if Daemonize is None:
raise ImportError(REQUIRES_DAEMONIZE_MESSAGE)
scripts/get_uwsgi_args.py
+116
View File
@@ -0,0 +1,116 @@
from __future__ import print_function
import os
import sys
from six import string_types
from six.moves import shlex_quote
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
from galaxy.util.path import get_ext
from galaxy.util.properties import load_app_properties, nice_config_parser
from galaxy.util.script import main_factory
DESCRIPTION = "Script to determine uWSGI command line arguments"
# socket is not an alias for http, but it is assumed that if you configure a socket in your uwsgi config you do not
# want to run the default http server (or you can configure it yourself)
ALIASES = {
'virtualenv': ('home', 'venv', 'pyhome'),
'pythonpath': ('python-path', 'pp'),
'http': ('httprouter', 'socket', 'uwsgi-socket', 'suwsgi-socket', 'ssl-socket'),
}
DEFAULT_ARGS = {
'_all_': ('virtualenv', 'pythonpath', 'master', 'threads', 'http', 'static-map', 'die-on-term', 'hook-master-start', 'enable-threads'),
'galaxy': ('py-call-osafterfork',),
'reports': (),
'tool_shed': (),
}
DEFAULT_PORTS = {
'galaxy': 8080,
'reports': 9001,
'tool_shed': 9009,
}
def __arg_set(arg, kwargs):
if arg in kwargs:
return True
for alias in ALIASES.get(arg, ()):
if alias in kwargs:
return True
return False
def __add_arg(args, arg, value):
optarg = '--%s' % arg
if isinstance(value, bool):
if value is True:
args.append(optarg)
elif isinstance(value, string_types):
# the = in --optarg=value is usually, but not always, optional
if value.startswith('='):
args.append(shlex_quote(optarg + value))
else:
args.append(optarg)
args.append(shlex_quote(value))
else:
[__add_arg(args, arg, v) for v in value]
def __add_config_file_arg(args, config_file, app):
ext = None
if config_file:
ext = get_ext(config_file)
if ext in ('yaml', 'json'):
__add_arg(args, ext, config_file)
elif ext == 'ini':
config = nice_config_parser(config_file)
has_logging = config.has_section('loggers')
if config.has_section('app:main'):
# uWSGI does not have any way to set the app name when loading with paste.deploy:loadapp(), so hardcoding
# the name to `main` is fine
__add_arg(args, 'ini-paste' if not has_logging else 'ini-paste-logged', config_file)
return # do not add --module
else:
__add_arg(args, ext, config_file)
if has_logging:
__add_arg(args, 'paste-logger', True)
__add_arg(args, 'module', 'galaxy.webapps.{app}.buildapp:uwsgi_app()'.format(app=app))
def _get_uwsgi_args(cliargs, kwargs):
# it'd be nice if we didn't have to reparse here but we need things out of more than one section
config_file = cliargs.config_file or kwargs.get('__file__')
uwsgi_kwargs = load_app_properties(config_file=config_file, config_section='uwsgi')
args = []
__add_config_file_arg(args, config_file, cliargs.app)
defaults = {
'virtualenv': os.environ.get('VIRTUAL_ENV', './.venv'),
'pythonpath': 'lib',
'master': True,
'threads': '4',
'http': 'localhost:{port}'.format(port=DEFAULT_PORTS[cliargs.app]),
'static-map': ('/static/style={here}/static/style/blue'.format(here=os.getcwd()),
'/static={here}/static'.format(here=os.getcwd())),
'die-on-term': True,
'enable-threads': True,
'hook-master-start': ('unix_signal:2 gracefully_kill_them_all',
'unix_signal:15 gracefully_kill_them_all'),
'py-call-osafterfork': True,
}
for arg in DEFAULT_ARGS['_all_'] + DEFAULT_ARGS[cliargs.app]:
if not __arg_set(arg, uwsgi_kwargs):
__add_arg(args, arg, defaults[arg])
print(' '.join(args))
ACTIONS = {
"get_uwsgi_args": _get_uwsgi_args,
}
if __name__ == '__main__':
main = main_factory(description=DESCRIPTION, actions=ACTIONS, default_action="get_uwsgi_args")
main()
scripts/manage_tools.py
-39
View File
@@ -1,39 +0,0 @@
from __future__ import print_function
import logging
import os.path
import sys
from ConfigParser import SafeConfigParser
from migrate.versioning.shell import main
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
from galaxy.model.orm.scripts import read_config_file_arg
from galaxy.util.properties import load_app_properties
log = logging.getLogger(__name__)
config_file = read_config_file_arg(sys.argv, 'config/galaxy.ini', 'universe_wsgi.ini')
if not os.path.exists(config_file):
print("Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % config_file)
sys.exit(1)
repo = 'lib/tool_shed/galaxy_install/migrate'
properties = load_app_properties(config_file=config_file)
cp = SafeConfigParser()
cp.read(config_file)
if config_file == 'config/galaxy.ini.sample' and 'GALAXY_TEST_DBURI' in os.environ:
# Running functional tests.
db_url = os.environ['GALAXY_TEST_DBURI']
elif "install_database_connection" in properties:
db_url = properties["install_database_connection"]
elif "database_connection" in properties:
db_url = properties["database_connection"]
elif "database_file" in properties:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % properties["database_file"]
else:
db_url = "sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE"
main(repository=repo, url=db_url)
static/scripts/bundled/admin.bundled.js
+2 -2
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/adminToolshed.bundled.js
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/analysis.bundled.js
+3 -3
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/library.bundled.js
+2 -2
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/libs.bundled.js
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/login.bundled.js
+2 -2
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/masthead.bundled.js
+2 -2
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/viz.bundled.js
+2 -2
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/workflow.bundled.js
+2 -2
View File
File diff suppressed because one or more lines are too long
static/scripts/layout/menu.js
+1 -1
View File
File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More