From dd65aeef719ac79317ebf84c3f6f7773d0bf4333 Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Fri, 29 May 2020 10:46:59 -0400 Subject: [PATCH 001/188] Fix null node - to do: node.app is still null --- .../scripts/components/Workflow/Editor/Node.vue | 14 +++++++++++--- .../components/Workflow/Editor/Recommendations.vue | 7 ++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index 5baff6c3a66..acb37272b55 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -167,6 +167,9 @@ export default { return getGalaxyInstance().config.enable_tool_recommendations; }, }, + created() { //might not need this later + this.onCreate(); + }, methods: { onChange() { this.manager.nodeChanged(this); @@ -193,15 +196,20 @@ export default { this.activeOutputs.toggle(name); this.manager.has_changes = true; }, - onCreate(toolId, event) { + //onCreate(toolId, event) { + onCreate(event) { + const toolId = this.contentId; const requestData = { tool_id: toolId, type: "tool", _: "true", }; + console.log("\n\n\nON CREATE"); getModule(requestData).then((response) => { - const node = this.manager.createNode("tool", response.name, toolId); - this.manager.setNode(node, response); + console.log("\n\n\nNODE.VUE"); + console.log("\n\n\nTOOL ID", toolId); + this.node = this.manager.createNode("tool", response.name, toolId); + this.manager.setNode(this.node, response); this.popoverShow = false; }); }, diff --git a/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue b/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue index c4d7176d848..d039ff8cf45 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue @@ -103,12 +103,17 @@ export default { return stepNameList.join(","); }, loadRecommendations() { - const workflowSimple = this.node.app.to_simple(); + console.log ("\n\nLOAD RECS: "); + console.log ("\n\nNODE: ", this.node); + console.log ("\n\nNODE.APP: ", this.node.app); + const workflowSimple = this.node.app.to_simple(); //source of error, this.node is null const node = this.node; const toolSequence = this.getWorkflowPath(workflowSimple, node.id); const requestData = { tool_sequence: toolSequence }; + console.log ("\n\nREQUEST DATA: ", predictedData); getToolPredictions(requestData).then((responsePred) => { const predictedData = responsePred.predicted_data; + console.log ("\n\nPREDICTED DATA: ", predictedData); const outputDatatypes = predictedData.o_extensions; const predictedDataChildren = predictedData.children; const app = this.node.app; From f962f1a3650e48f881aa7f33f5b34ce737b1bed0 Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Mon, 1 Jun 2020 01:33:27 -0400 Subject: [PATCH 002/188] Solve null node and null node.app -Solve manager vs app confusion -Prettier --- .../components/Workflow/Editor/Node.vue | 20 ++++------------ .../Workflow/Editor/Recommendations.vue | 24 +++++++------------ 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index acb37272b55..eafff88a2ac 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -23,7 +23,7 @@ - + {{ title }} - - {{ errors }} - + {{ errors }}
{ - console.log("\n\n\nNODE.VUE"); - console.log("\n\n\nTOOL ID", toolId); - this.node = this.manager.createNode("tool", response.name, toolId); - this.manager.setNode(this.node, response); + const node = this.manager.createNode("tool", response.name, toolId); + this.manager.setNode(node, response); this.popoverShow = false; }); }, diff --git a/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue b/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue index d039ff8cf45..35625a71ed2 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue @@ -7,17 +7,13 @@ -
- {{ deprecatedMessage }} -
-
- {{ noRecommendationsMessage }} -
+
{{ deprecatedMessage }}
+
{{ noRecommendationsMessage }}
@@ -103,20 +99,16 @@ export default { return stepNameList.join(","); }, loadRecommendations() { - console.log ("\n\nLOAD RECS: "); - console.log ("\n\nNODE: ", this.node); - console.log ("\n\nNODE.APP: ", this.node.app); - const workflowSimple = this.node.app.to_simple(); //source of error, this.node is null + const manager = this.node.manager; + const workflowSimple = manager.toSimple(); const node = this.node; const toolSequence = this.getWorkflowPath(workflowSimple, node.id); const requestData = { tool_sequence: toolSequence }; - console.log ("\n\nREQUEST DATA: ", predictedData); getToolPredictions(requestData).then((responsePred) => { const predictedData = responsePred.predicted_data; - console.log ("\n\nPREDICTED DATA: ", predictedData); const outputDatatypes = predictedData.o_extensions; const predictedDataChildren = predictedData.children; - const app = this.node.app; + const app = manager; this.isDeprecated = predictedData.is_deprecated; this.deprecatedMessage = predictedData.message; if (predictedDataChildren.length > 0) { From f85ca04a5834b2fcdfa70e0fb215b60c0f3ba371 Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Tue, 2 Jun 2020 01:13:10 -0400 Subject: [PATCH 003/188] PR comments: use get functions --- client/galaxy/scripts/components/Workflow/Editor/Node.vue | 2 +- .../scripts/components/Workflow/Editor/Recommendations.vue | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index eafff88a2ac..e1d7277c985 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -23,7 +23,7 @@ - + { From 1fed14fd00c8db1b9283aec203ced39f18b41db8 Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Tue, 16 Jun 2020 23:14:36 -0400 Subject: [PATCH 004/188] small style changes --- .../scripts/components/Workflow/Editor/Node.vue | 4 +++- .../components/Workflow/Editor/Recommendations.vue | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index e1d7277c985..2678eb54542 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -40,7 +40,9 @@ {{ title }} - {{ errors }} + + {{ errors }} +
-
{{ deprecatedMessage }}
-
{{ noRecommendationsMessage }}
+
+ {{ deprecatedMessage }} +
+
+ {{ noRecommendationsMessage }} +
From 9a8be72e5f5d8a895349af60a786fe67df6916b1 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Wed, 24 Jun 2020 01:25:05 +0530 Subject: [PATCH 005/188] Fix incorrect variable assignment --- lib/galaxy/web_stack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/web_stack/__init__.py b/lib/galaxy/web_stack/__init__.py index 6ea6c4a0aa5..c94a3064bf5 100644 --- a/lib/galaxy/web_stack/__init__.py +++ b/lib/galaxy/web_stack/__init__.py @@ -538,7 +538,7 @@ class WeblessApplicationStack(ApplicationStack): or (dialect.name == 'mysql' and dialect.server_version_info >= (8, 0, 1))): add_method = HANDLER_ASSIGNMENT_METHODS.DB_SKIP_LOCKED else: - HANDLER_ASSIGNMENT_METHODS.DB_TRANSACTION_ISOLATION + add_method = HANDLER_ASSIGNMENT_METHODS.DB_TRANSACTION_ISOLATION if add_method in job_config.UNSUPPORTED_HANDLER_ASSIGNMENT_METHODS: remove_methods.append(add_method) add_method = HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN From 46b5c959890709a1ad57d127438752ecc3a94b7f Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Wed, 24 Jun 2020 13:05:40 -0400 Subject: [PATCH 006/188] getNode() to getNode --- client/galaxy/scripts/components/Workflow/Editor/Node.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index 2678eb54542..590bd95f6e6 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -23,7 +23,7 @@ - + Date: Thu, 25 Jun 2020 11:44:44 -0400 Subject: [PATCH 007/188] Use getnode to access node, use datatypesmapping directly instead of global issubtype function --- .../components/Workflow/Editor/Node.vue | 2 +- .../Workflow/Editor/Recommendations.vue | 35 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index 590bd95f6e6..2c885a6372c 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -23,7 +23,7 @@ - + -
- {{ deprecatedMessage }} -
-
- {{ noRecommendationsMessage }} -
+
{{ deprecatedMessage }}
+
{{ noRecommendationsMessage }}
From 88a76cd65a9ddecd3b3f089363133a0072a3282f Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 18 Jun 2020 12:11:21 -0400 Subject: [PATCH 008/188] Added page documenting steps that happen during Galaxy startup. --- doc/source/dev/index.rst | 2 + doc/source/dev/starting_galaxy.md | 91 +++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 doc/source/dev/starting_galaxy.md diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index 0a459a371de..51888c12962 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -13,3 +13,5 @@ Development Documentation faq debugging_tests debugging_galaxy + starting_galaxy + diff --git a/doc/source/dev/starting_galaxy.md b/doc/source/dev/starting_galaxy.md new file mode 100644 index 00000000000..94a5d6ffcf5 --- /dev/null +++ b/doc/source/dev/starting_galaxy.md @@ -0,0 +1,91 @@ +# Starting Galaxy + +## What happens when you start Galaxy? + +Galaxy is started by running ./run.sh script. The following call graph documents what happens during Galaxy startup process. + +* ./galaxy/scripts/paster.py + * Add ./galaxy/lib to system path + * Check Python version + * version >= 3.5 -> OK + * version = 2.7 -> Deprecated + * No other version is supported + * serve.run() + +* ./galaxy/lib/galaxy/util/pastescript/serve.py (The Python Paste package contains Python modules that help in implementing WSGI middleware) + * serve.run() + * Parse Python command line arguments + * Extract the command name from command line arguments (e.g., 'serve') + * Get the Command object for command name, then invoke it (call Command's run() method) + * Command.run() + * Setup default options, validate command line arguments, then call ServeCommand's command() method + * ServeCommand.command() + * Conditionally, stop the daemon + * If a config file is required, make sure a config file is provided on the command line arguments. + * Possible subcommands: start, stop, restart, status (or None) + * Call loadserver() method in ./galaxy/lib/galaxy/util/pastescript/loadwsgi.py + * Calls loadobj() -> calls LoaderContext's create() -> calls _Server's invoke() + * Call loadapp() method in ./galaxy/lib/galaxy/util/pastescript/loadwsgi.py + * Calls loadobj() -> calls LoaderContext's create() -> calls _App's invoke() -> calls fix_call() on app_factory (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * app_factory: + 1. Create UniverseApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * Create GalaxyAppConfiguration instance from config file specified on the command line, then check for errors + * Configure logging (in ./galaxy/lib/galaxy/config/__init__.py) + * Create ExecutionTimerFactory + * Which has a GalaxyStatsdClient field + * Configure FluentTraceLogger + * Create the application stack (uWSGi, Paste, or Webless) + * App registers post fork functions to start application stack threads + * Create GalaxyQueueWorker instance (a worker for Galaxy's queues, used for dispatching control tasks. Sqlite instance) + * App registers post fork function to start queue worker threads + * Configure tool shed registery + * Configure object store. build_object_store_from_config() (in ./galaxy/lib/galaxy/objectstore/__init__.py) calls objectstore_class.from_xml(), calls clazz.parse_xml() + * Configure models in _configure_models() + * Calls create_or_verify_database() in ./galaxy/libl/galaxy/model/migrate/check.py + * If a new database, calls migrate_from_scratch() in ./galaxy/libl/galaxy/model/migrate/check.py + * Calls mapping.init() in ./galaxy/libl/galaxy/model/mapping.py (Details of how data objects are mapped to relational DB tables). + * Calls init_models_from_config in in ./galaxy/libl/galaxy/model/mapping.py + * Configure security + * Set GalaxyTagHandler (in ./galaxy/lib/galaxy/model/tags.py) + * Handles CRUD operations for tagging datasets, workflows, visualizations, etc + * Set DatasetCollectionManager (in ./galaxy/lib/galaxy/managers/collections.py) + * Configure HistoryManager + * Configure HistoryDatasetAssociation (HDA) Manager + * Create WorkflowsManager + * Handles CRUD operations related to workflows + * Configure DependencyResolverView (a RESTful interface to galaxy.tool_util.deps.DependencyResolver) + * Configure TestDataResolver + * Configure LibraryManager (Data libraries under Shared Data on use galaxy.org) + * Configure FolderManager (Folders are folders in a data library) + * Configure DynamicToolManager (Tools that are loaded from DB instead of disk files) + * Configure DataProviderRegistry + * Configure JobMetrics manager + * Configure ErrorReport plugins + * Configure job management configuration + * Setup a tool cache + * Watch various config files for immediate reload + * Configure Data Manager + * Configure UpdateREspositoyManager + * Load proprietary datatype convertors and display applications + * Configure VisualizationRegistry, ToursRegistry, and WebHookRegistry + * Load security policy and quota management + * Configure heartbeat for thread profiling + * If configured, create Sentry client + * App registers post fork functions to start Sentry client thread + * Create JobManager + * App registers post fork functions to start application job manager thread + * Job manager thread starts monitor threads for JobHandlerQueue and JobHandlerStopQueue + * JobManager has a JobHandler field, which manages the preparation, running, tracking, and finishing of jobs + * JobHandler has a job dispatcher, which launches underlying job runners + * DefaultJobDispatcher calls app's JobConfiguration's get_job_runner_plugins() to create LocalJobRunner instances + * The default number of LocalJobRuner instances is 4 + * JobHandler also has queues for starting and stopping jobs (unless track_jobs_in_database is True) + * Create WorkflowSchedulingManager + * Starts workflow request monitor thread + * Configure InteractiveToolManager + * Create database heartbeat instance + * App registers post fork functions to start application DB heartbeat thread + 2. Create CommunityWebApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * CommunityWebApplication: A WSGI application that maps requests to objects using routes and to methods on those objects + * Map all requests to controllers/actions + * Wrap web app in some middleware and return webapp From ef38ff5390911aa409b42101c87fa1964e03db80 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 18 Jun 2020 12:18:59 -0400 Subject: [PATCH 009/188] Minor revision --- doc/source/dev/starting_galaxy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/dev/starting_galaxy.md b/doc/source/dev/starting_galaxy.md index 94a5d6ffcf5..5f33e1f230f 100644 --- a/doc/source/dev/starting_galaxy.md +++ b/doc/source/dev/starting_galaxy.md @@ -28,7 +28,7 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Call loadapp() method in ./galaxy/lib/galaxy/util/pastescript/loadwsgi.py * Calls loadobj() -> calls LoaderContext's create() -> calls _App's invoke() -> calls fix_call() on app_factory (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) * app_factory: - 1. Create UniverseApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * Create UniverseApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) * Create GalaxyAppConfiguration instance from config file specified on the command line, then check for errors * Configure logging (in ./galaxy/lib/galaxy/config/__init__.py) * Create ExecutionTimerFactory @@ -85,7 +85,7 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Configure InteractiveToolManager * Create database heartbeat instance * App registers post fork functions to start application DB heartbeat thread - 2. Create CommunityWebApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * Create CommunityWebApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) * CommunityWebApplication: A WSGI application that maps requests to objects using routes and to methods on those objects * Map all requests to controllers/actions * Wrap web app in some middleware and return webapp From ed30e7b0517b5e86accaa8ebe70d86e9dc1f9f4c Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 18 Jun 2020 15:13:18 -0400 Subject: [PATCH 010/188] More small revision --- doc/source/dev/starting_galaxy.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/source/dev/starting_galaxy.md b/doc/source/dev/starting_galaxy.md index 5f33e1f230f..455c922bf76 100644 --- a/doc/source/dev/starting_galaxy.md +++ b/doc/source/dev/starting_galaxy.md @@ -37,7 +37,6 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Create the application stack (uWSGi, Paste, or Webless) * App registers post fork functions to start application stack threads * Create GalaxyQueueWorker instance (a worker for Galaxy's queues, used for dispatching control tasks. Sqlite instance) - * App registers post fork function to start queue worker threads * Configure tool shed registery * Configure object store. build_object_store_from_config() (in ./galaxy/lib/galaxy/objectstore/__init__.py) calls objectstore_class.from_xml(), calls clazz.parse_xml() * Configure models in _configure_models() @@ -65,7 +64,7 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Setup a tool cache * Watch various config files for immediate reload * Configure Data Manager - * Configure UpdateREspositoyManager + * Configure UpdateRespositoyManager * Load proprietary datatype convertors and display applications * Configure VisualizationRegistry, ToursRegistry, and WebHookRegistry * Load security policy and quota management @@ -74,17 +73,17 @@ Galaxy is started by running ./run.sh script. The following call graph documents * App registers post fork functions to start Sentry client thread * Create JobManager * App registers post fork functions to start application job manager thread - * Job manager thread starts monitor threads for JobHandlerQueue and JobHandlerStopQueue + * Job manager thread starts monitor threads for JobHandlerQueue and JobHandlerStopQueue (2 threads) * JobManager has a JobHandler field, which manages the preparation, running, tracking, and finishing of jobs * JobHandler has a job dispatcher, which launches underlying job runners * DefaultJobDispatcher calls app's JobConfiguration's get_job_runner_plugins() to create LocalJobRunner instances - * The default number of LocalJobRuner instances is 4 + * The default number of LocalJobRuner instances is 4 (4 threads) * JobHandler also has queues for starting and stopping jobs (unless track_jobs_in_database is True) * Create WorkflowSchedulingManager - * Starts workflow request monitor thread + * Starts workflow request monitor thread in __start_request_monitor() method (1 thread) * Configure InteractiveToolManager * Create database heartbeat instance - * App registers post fork functions to start application DB heartbeat thread + * App registers post fork functions to start application DB heartbeat thread (1 thread) * Create CommunityWebApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) * CommunityWebApplication: A WSGI application that maps requests to objects using routes and to methods on those objects * Map all requests to controllers/actions From b2ad77340aea77b3a7f0d3307c6a0a2f89e766a2 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Fri, 19 Jun 2020 09:27:17 -0400 Subject: [PATCH 011/188] Revised doc based on PR feedback. --- doc/source/dev/starting_galaxy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/dev/starting_galaxy.md b/doc/source/dev/starting_galaxy.md index 455c922bf76..ab254604ce5 100644 --- a/doc/source/dev/starting_galaxy.md +++ b/doc/source/dev/starting_galaxy.md @@ -2,7 +2,7 @@ ## What happens when you start Galaxy? -Galaxy is started by running ./run.sh script. The following call graph documents what happens during Galaxy startup process. +Galaxy is started by running ./run.sh script. The following call graph documents what happens during Galaxy startup process. Please note, Galaxy is a WSGI app, so in principle all python WSGI servers can run the Galaxy app (paster, gunicorn and uwsgi are known to work and have some level of support). This doc is specific to Paste (This is also not the typical way Galaxy starts if you start it via run.sh which uses uwsgi by default). * ./galaxy/scripts/paster.py * Add ./galaxy/lib to system path From 2f6063c1f8d0c0f7d19a73357426a0b55a4cb9dc Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 25 Jun 2020 11:48:54 -0400 Subject: [PATCH 012/188] Revised documentation per code review feedback. --- doc/source/dev/starting_galaxy.md | 129 ++++++++++++++++-------------- 1 file changed, 67 insertions(+), 62 deletions(-) diff --git a/doc/source/dev/starting_galaxy.md b/doc/source/dev/starting_galaxy.md index ab254604ce5..e3a2d556c66 100644 --- a/doc/source/dev/starting_galaxy.md +++ b/doc/source/dev/starting_galaxy.md @@ -2,7 +2,9 @@ ## What happens when you start Galaxy? -Galaxy is started by running ./run.sh script. The following call graph documents what happens during Galaxy startup process. Please note, Galaxy is a WSGI app, so in principle all python WSGI servers can run the Galaxy app (paster, gunicorn and uwsgi are known to work and have some level of support). This doc is specific to Paste (This is also not the typical way Galaxy starts if you start it via run.sh which uses uwsgi by default). +Galaxy is started by running ./run.sh script. The following call graph documents what happens during Galaxy startup process. Please note, Galaxy is a WSGI app, so in principle all Python WSGI servers can run the Galaxy app (Paste, Gunicorn and uWSGI are known to work and have some level of support). This document separates web server specific section (e.g. Paste) from Galaxy specific section (FYI, Galaxy uses uWSGI by default). + +## Paste * ./galaxy/scripts/paster.py * Add ./galaxy/lib to system path @@ -27,64 +29,67 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Calls loadobj() -> calls LoaderContext's create() -> calls _Server's invoke() * Call loadapp() method in ./galaxy/lib/galaxy/util/pastescript/loadwsgi.py * Calls loadobj() -> calls LoaderContext's create() -> calls _App's invoke() -> calls fix_call() on app_factory (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) - * app_factory: - * Create UniverseApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) - * Create GalaxyAppConfiguration instance from config file specified on the command line, then check for errors - * Configure logging (in ./galaxy/lib/galaxy/config/__init__.py) - * Create ExecutionTimerFactory - * Which has a GalaxyStatsdClient field - * Configure FluentTraceLogger - * Create the application stack (uWSGi, Paste, or Webless) - * App registers post fork functions to start application stack threads - * Create GalaxyQueueWorker instance (a worker for Galaxy's queues, used for dispatching control tasks. Sqlite instance) - * Configure tool shed registery - * Configure object store. build_object_store_from_config() (in ./galaxy/lib/galaxy/objectstore/__init__.py) calls objectstore_class.from_xml(), calls clazz.parse_xml() - * Configure models in _configure_models() - * Calls create_or_verify_database() in ./galaxy/libl/galaxy/model/migrate/check.py - * If a new database, calls migrate_from_scratch() in ./galaxy/libl/galaxy/model/migrate/check.py - * Calls mapping.init() in ./galaxy/libl/galaxy/model/mapping.py (Details of how data objects are mapped to relational DB tables). - * Calls init_models_from_config in in ./galaxy/libl/galaxy/model/mapping.py - * Configure security - * Set GalaxyTagHandler (in ./galaxy/lib/galaxy/model/tags.py) - * Handles CRUD operations for tagging datasets, workflows, visualizations, etc - * Set DatasetCollectionManager (in ./galaxy/lib/galaxy/managers/collections.py) - * Configure HistoryManager - * Configure HistoryDatasetAssociation (HDA) Manager - * Create WorkflowsManager - * Handles CRUD operations related to workflows - * Configure DependencyResolverView (a RESTful interface to galaxy.tool_util.deps.DependencyResolver) - * Configure TestDataResolver - * Configure LibraryManager (Data libraries under Shared Data on use galaxy.org) - * Configure FolderManager (Folders are folders in a data library) - * Configure DynamicToolManager (Tools that are loaded from DB instead of disk files) - * Configure DataProviderRegistry - * Configure JobMetrics manager - * Configure ErrorReport plugins - * Configure job management configuration - * Setup a tool cache - * Watch various config files for immediate reload - * Configure Data Manager - * Configure UpdateRespositoyManager - * Load proprietary datatype convertors and display applications - * Configure VisualizationRegistry, ToursRegistry, and WebHookRegistry - * Load security policy and quota management - * Configure heartbeat for thread profiling - * If configured, create Sentry client - * App registers post fork functions to start Sentry client thread - * Create JobManager - * App registers post fork functions to start application job manager thread - * Job manager thread starts monitor threads for JobHandlerQueue and JobHandlerStopQueue (2 threads) - * JobManager has a JobHandler field, which manages the preparation, running, tracking, and finishing of jobs - * JobHandler has a job dispatcher, which launches underlying job runners - * DefaultJobDispatcher calls app's JobConfiguration's get_job_runner_plugins() to create LocalJobRunner instances - * The default number of LocalJobRuner instances is 4 (4 threads) - * JobHandler also has queues for starting and stopping jobs (unless track_jobs_in_database is True) - * Create WorkflowSchedulingManager - * Starts workflow request monitor thread in __start_request_monitor() method (1 thread) - * Configure InteractiveToolManager - * Create database heartbeat instance - * App registers post fork functions to start application DB heartbeat thread (1 thread) - * Create CommunityWebApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) - * CommunityWebApplication: A WSGI application that maps requests to objects using routes and to methods on those objects - * Map all requests to controllers/actions - * Wrap web app in some middleware and return webapp + +## Galaxy + +* app_factory: + * Create UniverseApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * Create GalaxyAppConfiguration instance from config file specified on the command line, then check for errors + * Configure logging (in ./galaxy/lib/galaxy/config/__init__.py) + * Create ExecutionTimerFactory + * Which has a GalaxyStatsdClient field + * Configure FluentTraceLogger + * Create the application stack (uWSGi, Paste, or Webless) + * App registers post fork functions to start application stack threads + * Create GalaxyQueueWorker instance (a worker for Galaxy's queues, used for dispatching control tasks. Sqlite instance) + * Configure tool shed registry + * Configure object store. build_object_store_from_config() (in ./galaxy/lib/galaxy/objectstore/__init__.py) calls objectstore_class.from_xml(), calls clazz.parse_xml() + * Configure models in _configure_models() + * Calls create_or_verify_database() in ./galaxy/libl/galaxy/model/migrate/check.py + * If a new database, calls migrate_from_scratch() in ./galaxy/libl/galaxy/model/migrate/check.py + * Calls mapping.init() in ./galaxy/libl/galaxy/model/mapping.py (Details of how data objects are mapped to relational DB tables). + * Calls init_models_from_config in in ./galaxy/libl/galaxy/model/mapping.py + * Configure security + * Set GalaxyTagHandler (in ./galaxy/lib/galaxy/model/tags.py) + * Handles CRUD operations for tagging datasets, workflows, visualizations, etc + * Set DatasetCollectionManager (in ./galaxy/lib/galaxy/managers/collections.py) + * Configure HistoryManager + * Configure HistoryDatasetAssociation (HDA) Manager + * Create WorkflowsManager + * Handles CRUD operations related to workflows + * Configure DependencyResolverView (a RESTful interface to galaxy.tool_util.deps.DependencyResolver) + * Configure TestDataResolver + * Configure LibraryManager (Data libraries under Shared Data) + * Configure FolderManager (Folders are folders in a data library) + * Configure DynamicToolManager (Tools that are loaded from DB instead of disk files) + * Configure DataProviderRegistry + * Configure JobMetrics manager + * Configure ErrorReport plugins + * Configure job management configuration + * Setup a tool cache + * Watch various config files for immediate reload + * Configure Data Manager + * Configure UpdateRespositoyManager + * Load proprietary datatype convertors and display applications + * Configure VisualizationRegistry, ToursRegistry, and WebHookRegistry + * Load security policy and quota management + * Configure heartbeat for thread profiling + * If configured, create Sentry client + * App registers post fork functions to start Sentry client thread + * Create JobManager + * App registers post fork functions to start application job manager thread + * Job manager thread starts monitor threads for JobHandlerQueue and JobHandlerStopQueue (2 threads) + * JobManager has a JobHandler field, which manages the preparation, running, tracking, and finishing of jobs + * JobHandler has a job dispatcher, which launches underlying job runners + * DefaultJobDispatcher calls app's JobConfiguration's get_job_runner_plugins() to create LocalJobRunner instances + * The default number of LocalJobRuner instances is 4 (4 threads) + * JobHandler also has queues for starting and stopping jobs (unless track_jobs_in_database is True) + * Create WorkflowSchedulingManager + * Starts workflow request monitor thread in __start_request_monitor() method (1 thread) + * Configure InteractiveToolManager + * Create database heartbeat instance + * App registers post fork functions to start application DB heartbeat thread (1 thread) + * Create CommunityWebApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * CommunityWebApplication: A WSGI application that maps requests to objects using routes and to methods on those objects + * Map all requests to controllers/actions + * Wrap web app in some middleware and return webapp From 072be091e2787861f9056903d9d7e5e4427da390 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 25 Jun 2020 15:31:39 -0400 Subject: [PATCH 013/188] Removed trailing whitespace --- doc/source/dev/index.rst | 1 - doc/source/dev/starting_galaxy.md | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index 51888c12962..dfc9f93430a 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -14,4 +14,3 @@ Development Documentation debugging_tests debugging_galaxy starting_galaxy - diff --git a/doc/source/dev/starting_galaxy.md b/doc/source/dev/starting_galaxy.md index e3a2d556c66..2f8c483f45e 100644 --- a/doc/source/dev/starting_galaxy.md +++ b/doc/source/dev/starting_galaxy.md @@ -22,16 +22,16 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Command.run() * Setup default options, validate command line arguments, then call ServeCommand's command() method * ServeCommand.command() - * Conditionally, stop the daemon + * Conditionally, stop the daemon * If a config file is required, make sure a config file is provided on the command line arguments. * Possible subcommands: start, stop, restart, status (or None) * Call loadserver() method in ./galaxy/lib/galaxy/util/pastescript/loadwsgi.py - * Calls loadobj() -> calls LoaderContext's create() -> calls _Server's invoke() + * Calls loadobj() -> calls LoaderContext's create() -> calls _Server's invoke() * Call loadapp() method in ./galaxy/lib/galaxy/util/pastescript/loadwsgi.py * Calls loadobj() -> calls LoaderContext's create() -> calls _App's invoke() -> calls fix_call() on app_factory (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) -## Galaxy - +## Galaxy + * app_factory: * Create UniverseApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) * Create GalaxyAppConfiguration instance from config file specified on the command line, then check for errors @@ -40,11 +40,11 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Which has a GalaxyStatsdClient field * Configure FluentTraceLogger * Create the application stack (uWSGi, Paste, or Webless) - * App registers post fork functions to start application stack threads + * App registers post fork functions to start application stack threads * Create GalaxyQueueWorker instance (a worker for Galaxy's queues, used for dispatching control tasks. Sqlite instance) * Configure tool shed registry * Configure object store. build_object_store_from_config() (in ./galaxy/lib/galaxy/objectstore/__init__.py) calls objectstore_class.from_xml(), calls clazz.parse_xml() - * Configure models in _configure_models() + * Configure models in _configure_models() * Calls create_or_verify_database() in ./galaxy/libl/galaxy/model/migrate/check.py * If a new database, calls migrate_from_scratch() in ./galaxy/libl/galaxy/model/migrate/check.py * Calls mapping.init() in ./galaxy/libl/galaxy/model/mapping.py (Details of how data objects are mapped to relational DB tables). @@ -52,7 +52,7 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Configure security * Set GalaxyTagHandler (in ./galaxy/lib/galaxy/model/tags.py) * Handles CRUD operations for tagging datasets, workflows, visualizations, etc - * Set DatasetCollectionManager (in ./galaxy/lib/galaxy/managers/collections.py) + * Set DatasetCollectionManager (in ./galaxy/lib/galaxy/managers/collections.py) * Configure HistoryManager * Configure HistoryDatasetAssociation (HDA) Manager * Create WorkflowsManager @@ -77,8 +77,8 @@ Galaxy is started by running ./run.sh script. The following call graph documents * If configured, create Sentry client * App registers post fork functions to start Sentry client thread * Create JobManager - * App registers post fork functions to start application job manager thread - * Job manager thread starts monitor threads for JobHandlerQueue and JobHandlerStopQueue (2 threads) + * App registers post fork functions to start application job manager thread + * Job manager thread starts monitor threads for JobHandlerQueue and JobHandlerStopQueue (2 threads) * JobManager has a JobHandler field, which manages the preparation, running, tracking, and finishing of jobs * JobHandler has a job dispatcher, which launches underlying job runners * DefaultJobDispatcher calls app's JobConfiguration's get_job_runner_plugins() to create LocalJobRunner instances @@ -88,7 +88,7 @@ Galaxy is started by running ./run.sh script. The following call graph documents * Starts workflow request monitor thread in __start_request_monitor() method (1 thread) * Configure InteractiveToolManager * Create database heartbeat instance - * App registers post fork functions to start application DB heartbeat thread (1 thread) + * App registers post fork functions to start application DB heartbeat thread (1 thread) * Create CommunityWebApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) * CommunityWebApplication: A WSGI application that maps requests to objects using routes and to methods on those objects * Map all requests to controllers/actions From 9b757c00405016ad77059332669230cf946e0c18 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 25 Jun 2020 22:47:31 +0100 Subject: [PATCH 014/188] Add original exception details in error message --- lib/galaxy/util/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 7dcce760886..e29a9227845 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1037,9 +1037,8 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=Fals # Now in Python 3, value is an instance of bytes or str if not isinstance(value, text_type): value = text_type(value, encoding, error) - except Exception: - msg = "Value '%s' could not be coerced to Unicode" % value - log.exception(msg) + except Exception as e: + msg = "Value '%s' could not be coerced to Unicode: %s('%s')" % (value, type(e).__name__, e) raise Exception(msg) if strip_null: return value.replace('\0', '') From b418b17bea0bc16e1c32cf95167246f81a1ce4c2 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 25 Jun 2020 17:49:23 -0400 Subject: [PATCH 015/188] Initial hack to fix cookie handling that changed with py3. Cookie handling used to be more robust, allowing invalid values to be set without breaking the whole thing. --- lib/galaxy/web/framework/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index edf1b22b976..9458f0911cb 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -388,7 +388,7 @@ class Request(webob.Request): @lazy_property def cookies(self): - return get_cookies(self.environ) + return get_cookies({"HTTP_COOKIE": "; ".join([x.strip() for x in self.environ.get("HTTP_COOKIE").split('; ') if x.startswith('galaxy')])}) @lazy_property def base(self): From cb94255e5c8d446d471e6396f7d3022c1c0f2596 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 25 Jun 2020 18:28:51 -0400 Subject: [PATCH 016/188] Drop paste usage in cookie handling --- lib/galaxy/web/framework/base.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 9458f0911cb..8133cf1fec5 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -388,7 +388,16 @@ class Request(webob.Request): @lazy_property def cookies(self): - return get_cookies({"HTTP_COOKIE": "; ".join([x.strip() for x in self.environ.get("HTTP_COOKIE").split('; ') if x.startswith('galaxy')])}) + cookies = SimpleCookie() + cookie_header = self.environ.get("HTTP_COOKIE") + if cookie_header: + galaxy_cookies = "; ".join([x.strip() for x in cookie_header.split('; ') if x.startswith('galaxy')]) + if galaxy_cookies: + try: + cookies.load(galaxy_cookies) + except CookieError: + pass + return cookies @lazy_property def base(self): From 9b56449399e5356c999cc1f9cfcdec4e89a23126 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 25 Jun 2020 18:29:04 -0400 Subject: [PATCH 017/188] Fix, split imports --- lib/galaxy/web/framework/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 8133cf1fec5..e17e1b47238 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -16,9 +16,11 @@ import webob.compat import webob.exc import webob.exc as httpexceptions # noqa: F401 # We will use some very basic HTTP/wsgi utilities from the paste library -from paste.request import get_cookies from paste.response import HeaderDict -from six.moves.http_cookies import SimpleCookie +from six.moves.http_cookies import ( + CookieError, + SimpleCookie +) from galaxy.util import smart_str From e9c5aa84bd0ea77b47f126e960c1f564f2e7819d Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 25 Jun 2020 12:25:08 -0400 Subject: [PATCH 018/188] Fix import in recommendations component Fix workflow parsing to model helper Run prettier --- .../galaxy/scripts/components/Workflow/Editor/Node.vue | 7 ++++++- .../components/Workflow/Editor/Recommendations.vue | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/client/galaxy/scripts/components/Workflow/Editor/Node.vue b/client/galaxy/scripts/components/Workflow/Editor/Node.vue index 7b375382cc0..a5ba751f115 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/Node.vue +++ b/client/galaxy/scripts/components/Workflow/Editor/Node.vue @@ -23,7 +23,12 @@ - + import { getToolPredictions } from "./modules/services"; import LoadingSpan from "components/LoadingSpan"; -import { toSimple } from "module/model"; +import { toSimple } from "./modules/model"; import _l from "utils/localization"; export default { @@ -32,6 +32,10 @@ export default { type: Function, required: true, }, + getManager: { + type: Function, + required: true, + }, datatypesMapping: { type: Object, required: true, @@ -105,7 +109,8 @@ export default { }, loadRecommendations() { const node = this.getNode(); - const workflowSimple = toSimple(); + const workflow = this.getManager(); + const workflowSimple = toSimple(workflow); const toolSequence = this.getWorkflowPath(workflowSimple, node.id); const requestData = { tool_sequence: toolSequence }; getToolPredictions(requestData).then((responsePred) => { From 663b7931a15fbc90868e1d96fe9f7188c7630ec1 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 26 Jun 2020 01:01:13 +0100 Subject: [PATCH 019/188] Rewrite `check_html()` to read a chunk of each line Fix the following traceback during upload of files with very long lines (e.g. JSON from Ensembl): ``` Traceback (most recent call last): File "/srv/galaxy/tools/data_source/upload.py", line 326, in __main__() File "/srv/galaxy/tools/data_source/upload.py", line 319, in __main__ metadata.append(add_file(dataset, registry, output_path)) File "/srv/galaxy/tools/data_source/upload.py", line 128, in add_file convert_spaces_to_tabs=dataset.space_to_tab, File "/srv/galaxy/lib/galaxy/datatypes/upload_util.py", line 54, in handle_upload convert_spaces_to_tabs=convert_spaces_to_tabs, File "/srv/galaxy/lib/galaxy/datatypes/sniff.py", line 749, in handle_uploaded_dataset_file_internal if not is_binary and check_content and check_html(converted_path): File "/srv/galaxy/lib/galaxy/util/checkers.py", line 42, in check_html line = util.unicodify(line) File "/srv/galaxy/lib/galaxy/util/__init__.py", line 1042, in unicodify raise Exception(msg) Exception: Value '...' could not be coerced to Unicode: MemoryError('') ``` Also: - Use `with` statement to open/close files inside archives - Add tests --- lib/galaxy/util/checkers.py | 76 ++++++++++++++++----------------- test/unit/util/test_checkers.py | 19 +++++++++ 2 files changed, 57 insertions(+), 38 deletions(-) create mode 100644 test/unit/util/test_checkers.py diff --git a/lib/galaxy/util/checkers.py b/lib/galaxy/util/checkers.py index 3098d928705..6e64ce38d57 100644 --- a/lib/galaxy/util/checkers.py +++ b/lib/galaxy/util/checkers.py @@ -1,10 +1,14 @@ import gzip +import io import re import sys import tarfile import zipfile -from six import BytesIO +from six import ( + BytesIO, + StringIO +) from six.moves import filter from galaxy import util @@ -21,34 +25,35 @@ else: import bz2 HTML_CHECK_LINES = 100 +CHUNK_SIZE = 2 ** 15 # 32Kb +HTML_REGEXPS = ( + re.compile(r"]*HREF[^>]+>", re.I), + re.compile(r"]*>", re.I), + re.compile(r"]*>", re.I), + re.compile(r"]*>", re.I), + re.compile(r"]*>", re.I), +) -def check_html(file_path, chunk=None): - if chunk is None: - temp = open(file_path, mode='rb') - elif hasattr(chunk, "splitlines"): - temp = chunk.splitlines() +def check_html(name, file_path=True): + """ + Returns True if the file/string contains HTML code. + """ + # Handles files if file_path is True or text if file_path is False + if file_path: + temp = io.open(name, "r", encoding='utf-8') else: - temp = chunk - regexp1 = re.compile(r"]*HREF[^>]+>", re.I) - regexp2 = re.compile(r"]*>", re.I) - regexp3 = re.compile(r"]*>", re.I) - regexp4 = re.compile(r"]*>", re.I) - regexp5 = re.compile(r"]*>", re.I) - lineno = 0 - # TODO: Potentially reading huge lines into string here, this should be - # reworked. - for line in temp: - line = util.unicodify(line) - lineno += 1 - matches = regexp1.search(line) or regexp2.search(line) or regexp3.search(line) or regexp4.search(line) or regexp5.search(line) - if matches: - if chunk is None: - temp.close() - return True - if HTML_CHECK_LINES and (lineno > HTML_CHECK_LINES): - break - if chunk is None: + temp = StringIO(util.unicodify(name)) + try: + for _ in range(HTML_CHECK_LINES): + line = temp.readline(CHUNK_SIZE) + if not line: + break + if any(regexp.search(line) for regexp in HTML_REGEXPS): + return True + except UnicodeDecodeError: + return False + finally: temp.close() return False @@ -89,12 +94,10 @@ def check_gzip(file_path, check_content=True): if not check_content: return (True, True) - CHUNK_SIZE = 2 ** 15 # 32Kb - gzipped_file = gzip.GzipFile(file_path, mode='rb') - chunk = gzipped_file.read(CHUNK_SIZE) - gzipped_file.close() + with gzip.open(file_path, mode='rb') as gzipped_file: + chunk = gzipped_file.read(CHUNK_SIZE) # See if we have a compressed HTML file - if check_html(file_path, chunk=chunk): + if check_html(chunk, file_path=False): return (True, False) return (True, True) @@ -111,12 +114,10 @@ def check_bz2(file_path, check_content=True): if not check_content: return (True, True) - CHUNK_SIZE = 2 ** 15 # reKb - bzipped_file = bz2.BZ2File(file_path, mode='rb') - chunk = bzipped_file.read(CHUNK_SIZE) - bzipped_file.close() + with bz2.BZ2File(file_path, mode='rb') as bzipped_file: + chunk = bzipped_file.read(CHUNK_SIZE) # See if we have a compressed HTML file - if check_html(file_path, chunk=chunk): + if check_html(chunk, file_path=False): return (True, False) return (True, True) @@ -128,12 +129,11 @@ def check_zip(file_path, check_content=True, files=1): if not check_content: return (True, True) - CHUNK_SIZE = 2 ** 15 # 32Kb chunk = None for filect, member in enumerate(iter_zip(file_path)): handle, name = member chunk = handle.read(CHUNK_SIZE) - if chunk and check_html(file_path, chunk): + if chunk and check_html(chunk, file_path=False): return (True, False) if filect >= files: break diff --git a/test/unit/util/test_checkers.py b/test/unit/util/test_checkers.py new file mode 100644 index 00000000000..5a01b45dcb9 --- /dev/null +++ b/test/unit/util/test_checkers.py @@ -0,0 +1,19 @@ +import tempfile + +from galaxy.util.checkers import check_html + + +def test_check_html(): + html_text = '

\nLink\n

\n' + assert check_html(html_text, file_path=False) + # Test a non-HTML binary string + assert not check_html(b'No HTML here\nSecond line\n', file_path=False) + with tempfile.NamedTemporaryFile(mode='w') as tmp: + tmp.write(html_text) + tmp.flush() + assert check_html(tmp.name) + # Test a non-UTF8 binary file + with tempfile.NamedTemporaryFile(mode='wb') as tmp: + tmp.write(b'\x1f\x8b') + tmp.flush() + assert not check_html(tmp.name) From 3d2eae565a5e70184b2a5b324117a8bcfa1e9d94 Mon Sep 17 00:00:00 2001 From: Dannon Date: Fri, 26 Jun 2020 07:59:11 -0400 Subject: [PATCH 020/188] Update lib/galaxy/web/framework/base.py Co-authored-by: Nicola Soranzo --- lib/galaxy/web/framework/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index e17e1b47238..af50ddb9773 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -393,7 +393,7 @@ class Request(webob.Request): cookies = SimpleCookie() cookie_header = self.environ.get("HTTP_COOKIE") if cookie_header: - galaxy_cookies = "; ".join([x.strip() for x in cookie_header.split('; ') if x.startswith('galaxy')]) + galaxy_cookies = "; ".join(x.strip() for x in cookie_header.split('; ') if x.startswith('galaxy')) if galaxy_cookies: try: cookies.load(galaxy_cookies) From 975ba9166d1b53bda0a678a97a8d76d4d043ea4d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 26 Jun 2020 18:14:46 +0100 Subject: [PATCH 021/188] Read a chunk of each line in `Text.count_data_lines()` Prevent out-of-memory issues when setting metadata of files with very long lines, e.g. ftp://ftp.ensembl.org/pub/current_json/homo_sapiens/homo_sapiens.json --- lib/galaxy/datatypes/data.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index cbb139a3ac0..3df7b20f3d3 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -179,12 +179,14 @@ class Data(object): """Unimplemented method, allows guessing of metadata from contents of file""" return True - def missing_meta(self, dataset, check=[], skip=[]): + def missing_meta(self, dataset, check=None, skip=None): """ Checks for empty metadata values, Returns True if non-optional metadata is missing Specifying a list of 'check' values will only check those names provided; when used, optionality is ignored Specifying a list of 'skip' items will return True even when a named metadata value is missing """ + if skip is None: + skip = [] if check: to_check = ((to_check, dataset.metadata.get(to_check)) for to_check in check) else: @@ -346,7 +348,7 @@ class Data(object): def __archive_extra_files_path(self, extra_files_path): """Yield filepaths and relative filepaths for files in extra_files_path""" - for root, dirs, files in os.walk(extra_files_path): + for root, _, files in os.walk(extra_files_path): for fname in files: fpath = os.path.join(root, fname) rpath = os.path.relpath(fpath, extra_files_path) @@ -859,13 +861,17 @@ class Text(Data): Count the number of lines of data in dataset, skipping all blank lines and comments. """ + CHUNK_SIZE = 2 ** 15 # 32Kb data_lines = 0 with compression_utils.get_fileobj(dataset.file_name) as in_file: # FIXME: Potential encoding issue can prevent the ability to iterate over lines # causing set_meta process to fail otherwise OK jobs. A better solution than # a silent try/except is desirable. try: - for line in in_file: + while True: + line = in_file.readline(CHUNK_SIZE) + if not line: + break line = line.strip() if line and not line.startswith('#'): data_lines += 1 From daa6715a568a924295f250d946e28f94604ebf57 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 26 Jun 2020 16:25:07 -0400 Subject: [PATCH 022/188] Enable regex validator to validate lists of values. --- lib/galaxy/tools/parameters/validation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index 8a6624bafe9..f75a31bc6be 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -79,8 +79,11 @@ class RegexValidator(Validator): self.expression = expression def validate(self, value, trans=None): - if re.match(self.expression, value or '') is None: - raise ValueError(self.message) + if not isinstance(value, list): + value = [value] + for val in value: + if re.match(self.expression, val or '') is None: + raise ValueError(self.message) class ExpressionValidator(Validator): From d384e155ad08ced5d7d8e865a32336de2be9189b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 26 Jun 2020 21:41:10 +0200 Subject: [PATCH 023/188] Speedup dataset discovery This reduces the flushes from N datasets + 2 to 3 flushes (if there are no tags to be crated, else it is 2*N datasets + 2 flushes) Creating 100 datasets via output discovery (splitting a text file with split_file_to_collection) With this commit: ``` *** PROFILER RESULTS *** discover_outputs (/Users/mvandenb/src/galaxy/lib/galaxy/jobs/__init__.py:1772) function called 1 times 792228 function calls (785314 primitive calls) in 1.009 seconds Ordered by: cumulative time, internal time, call count List reduced from 1266 to 40 due to restriction <40> ncalls tottime percall cumtime percall filename:lineno(function) 1 0.000 0.000 1.013 1.013 __init__.py:1772(discover_outputs) 1 0.000 0.000 1.013 1.013 __init__.py:1858(discover_outputs) 1 0.000 0.000 1.009 1.009 output_collect.py:87(collect_dynamic_outputs) 1 0.002 0.002 0.991 0.991 discover.py:192(populate_collection_elements) 1 0.000 0.000 0.442 0.442 output_collect.py:266(add_datasets_to_history) 1 0.000 0.000 0.441 0.441 __init__.py:1679(add_datasets) 818 0.001 0.000 0.375 0.000 base.py:916(execute) 818 0.001 0.000 0.373 0.000 elements.py:291(_execute_on_connection) 818 0.004 0.000 0.373 0.000 base.py:1052(_execute_clauseelement) 7984/7675 0.006 0.000 0.334 0.000 attributes.py:698(get) 3 0.001 0.000 0.330 0.110 session.py:2462(flush) 9022 0.005 0.000 0.330 0.000 attributes.py:278(__get__) 3 0.001 0.000 0.328 0.109 session.py:2515(_flush) 2009 0.002 0.000 0.320 0.000 strategies.py:665(_load_for_state) 100 0.002 0.000 0.316 0.003 discover.py:42(create_dataset) 818 0.006 0.000 0.306 0.000 base.py:1159(_execute_context) 3 0.000 0.000 0.292 0.097 unitofwork.py:402(execute) 1 0.000 0.000 0.246 0.246 __init__.py:1688() 100 0.000 0.000 0.246 0.002 __init__.py:2611(get_total_size) 19 0.000 0.000 0.224 0.012 persistence.py:184(save_obj) 817 0.001 0.000 0.206 0.000 default.py:589(do_execute) 817 0.201 0.000 0.206 0.000 {method 'execute' of 'psycopg2.extensions.cursor' objects} 314 0.001 0.000 0.190 0.001 query.py:3425(_execute_and_instances) 1 0.000 0.000 0.185 0.185 __init__.py:1700(__add_datasets_optimized) 19 0.005 0.000 0.184 0.010 persistence.py:1039(_emit_insert_statements) 1 0.000 0.000 0.181 0.181 mapping.py:2806(db_next_hid) 4/3 0.000 0.000 0.179 0.060 session.py:893(begin) 4/3 0.000 0.000 0.179 0.060 session.py:220(__init__) 4/3 0.000 0.000 0.179 0.060 session.py:337(_take_snapshot) 303 0.000 0.000 0.178 0.001 scoping.py:161(do) 600 0.002 0.000 0.177 0.000 __init__.py:269(_invoke) 207 0.000 0.000 0.175 0.001 state.py:640(_load_expired) 107 0.001 0.000 0.173 0.002 loading.py:938(load_scalar_attributes) 209 0.001 0.000 0.173 0.001 mapper.py:2797(_get_state_attr_by_column) 207 0.000 0.000 0.170 0.001 strategies.py:753(_get_ident_for_use_get) 107 0.000 0.000 0.170 0.002 loading.py:190(load_on_ident) 107 0.001 0.000 0.169 0.002 loading.py:211(load_on_pk_identity) 207 0.000 0.000 0.169 0.001 strategies.py:763() 107 0.000 0.000 0.168 0.002 query.py:3341(one) 107 0.002 0.000 0.168 0.002 query.py:3306(one_or_none) ``` Without fix: ``` *** PROFILER RESULTS *** discover_outputs (/Users/mvandenb/src/galaxy/lib/galaxy/jobs/__init__.py:1772) function called 1 times 1955870 function calls (1924658 primitive calls) in 2.463 seconds Ordered by: cumulative time, internal time, call count List reduced from 1289 to 40 due to restriction <40> ncalls tottime percall cumtime percall filename:lineno(function) 1 0.000 0.000 2.472 2.472 __init__.py:1772(discover_outputs) 1 0.000 0.000 2.472 2.472 __init__.py:1858(discover_outputs) 1 0.000 0.000 2.467 2.467 output_collect.py:87(collect_dynamic_outputs) 1 0.002 0.002 2.445 2.445 discover.py:190(populate_collection_elements) 100 0.002 0.000 1.682 0.017 discover.py:42(create_dataset) 11065/10557 0.009 0.000 1.151 0.000 attributes.py:698(get) 10222 0.007 0.000 1.146 0.000 attributes.py:278(__get__) 1616 0.003 0.000 1.020 0.001 base.py:916(execute) 1616 0.002 0.000 1.016 0.001 elements.py:291(_execute_on_connection) 1616 0.010 0.000 1.014 0.001 base.py:1052(_execute_clauseelement) 403 0.001 0.000 0.928 0.002 scoping.py:161(do) 103 0.004 0.000 0.905 0.009 session.py:2462(flush) 102 0.000 0.000 0.901 0.009 output_collect.py:212(flush) 103 0.003 0.000 0.900 0.009 session.py:2515(_flush) 505 0.002 0.000 0.812 0.002 state.py:640(_load_expired) 405 0.003 0.000 0.807 0.002 loading.py:938(load_scalar_attributes) 405 0.001 0.000 0.789 0.002 loading.py:190(load_on_ident) 405 0.003 0.000 0.788 0.002 loading.py:211(load_on_pk_identity) 405 0.000 0.000 0.782 0.002 query.py:3341(one) 405 0.008 0.000 0.781 0.002 query.py:3306(one_or_none) 812 0.003 0.000 0.713 0.001 query.py:3425(_execute_and_instances) 1616 0.014 0.000 0.709 0.000 base.py:1159(_execute_context) 103 0.003 0.000 0.708 0.007 unitofwork.py:402(execute) 2008 0.004 0.000 0.683 0.000 strategies.py:665(_load_for_state) 405 0.001 0.000 0.643 0.002 query.py:3400(__iter__) 1615 0.001 0.000 0.517 0.000 default.py:589(do_execute) 1615 0.504 0.000 0.516 0.000 {method 'execute' of 'psycopg2.extensions.cursor' objects} 715 0.007 0.000 0.435 0.001 persistence.py:184(save_obj) 408 0.002 0.000 0.368 0.001 mapper.py:2797(_get_state_attr_by_column) 306 0.001 0.000 0.360 0.001 strategies.py:753(_get_ident_for_use_get) 306 0.000 0.000 0.359 0.001 strategies.py:763() 101 0.000 0.000 0.342 0.003 output_collect.py:295(job_id) 407 0.001 0.000 0.316 0.001 :1() 407 0.005 0.000 0.315 0.001 strategies.py:772(_emit_lazyload) 615 0.001 0.000 0.313 0.001 unitofwork.py:585(execute) 418 0.001 0.000 0.287 0.001 elements.py:405(compile) 418 0.001 0.000 0.286 0.001 elements.py:470(_compiler) 418 0.002 0.000 0.285 0.001 compiler.py:527(__init__) 418 0.001 0.000 0.283 0.001 compiler.py:274(__init__) 426/418 0.000 0.000 0.282 0.001 compiler.py:349(process) ``` and that's having a larger impact with more datasets to discover (The covid workflows split a 10000 line file for instance). We can use this pattern in other places as well, but I prefer going through this one by one and opening separate PRs with profiling results. Ideally we should attempt to batch flushes for mapped-over jobs as well, which might increase our job throughout significantly. --- lib/galaxy/model/store/discover.py | 24 ++++++++----- lib/galaxy/model/tags.py | 57 ++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 0470cdd158f..36ee7428b8e 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -123,9 +123,11 @@ class ModelPersistenceContext(object): if created_from_basename is not None: primary_data.created_from_basename = created_from_basename - self.flush() - if tag_list: + # If we have a tag we need a primary id, so need to flush here + # TODO: eliminate creating tag associations within create dataset + # We can do this incrementally by not passing in a tag list. + self.flush() self.tag_handler.add_tags_from_list(self.job.user, primary_data, tag_list) # Move data from temp location to dataset location @@ -196,7 +198,7 @@ class ModelPersistenceContext(object): if name is None: name = "unnamed output" - element_datasets = [] + element_datasets = {'element_identifiers': [], 'datasets': [], 'tag_lists': []} for filename, discovered_file in filenames.items(): create_dataset_timer = ExecutionTimer() fields_match = discovered_file.match @@ -215,7 +217,6 @@ class ModelPersistenceContext(object): dataset_name = fields_match.name or designation link_data = discovered_file.match.link_data - tag_list = discovered_file.match.tag_list sources = discovered_file.match.sources hashes = discovered_file.match.hashes @@ -230,7 +231,6 @@ class ModelPersistenceContext(object): filename=filename, metadata_source_name=metadata_source_name, link_data=link_data, - tag_list=tag_list, sources=sources, hashes=hashes, created_from_basename=created_from_basename, @@ -244,10 +244,13 @@ class ModelPersistenceContext(object): name, create_dataset_timer, ) - element_datasets.append((element_identifiers, dataset)) + element_datasets['element_identifiers'].append(element_identifiers) + element_datasets['datasets'].append(dataset) + element_datasets['tag_lists'].append(discovered_file.match.tag_list) add_datasets_timer = ExecutionTimer() - self.add_datasets_to_history([d for (ei, d) in element_datasets]) + self.add_datasets_to_history(element_datasets['datasets']) + self.add_tags_to_datasets(datasets=element_datasets['datasets'], tag_lists=element_datasets['tag_lists']) log.debug( "(%s) Add dynamic collection datasets to history for output [%s] %s", self.job_id(), @@ -255,7 +258,7 @@ class ModelPersistenceContext(object): add_datasets_timer, ) - for (element_identifiers, dataset) in element_datasets: + for (element_identifiers, dataset) in zip(element_datasets['element_identifiers'], element_datasets['datasets']): current_builder = root_collection_builder for element_identifier in element_identifiers[:-1]: current_builder = current_builder.get_level(element_identifier) @@ -268,6 +271,11 @@ class ModelPersistenceContext(object): self.flush() + def add_tags_to_datasets(self, datasets, tag_lists): + tag_session = self.tag_handler.create_tag_handler_session() + for dataset, tags in zip(datasets, tag_lists): + tag_session.add_tags_from_list(self.job.user, dataset, tags, flush=False) + @abc.abstractproperty def tag_handler(self): """Return a galaxy.model.tags.TagHandler-like object for persisting tags.""" diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 33c48750192..917d81d2abd 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -42,26 +42,31 @@ class TagHandler(object): # Initialize with known classes - add to this in subclasses. self.item_tag_assoc_info = {} - def add_tags_from_list(self, user, item, new_tags_list): + def create_tag_handler_session(self): + # Creates a transient tag handler that avoids repeated flushes + return GalaxyTagHandlerSession(self.sa_session) + + def add_tags_from_list(self, user, item, new_tags_list, flush=True): new_tags_set = set(new_tags_list) if item.tags: new_tags_set.update(self.get_tags_str(item.tags).split(',')) - return self.set_tags_from_list(user, item, new_tags_set) + return self.set_tags_from_list(user, item, new_tags_set, flush=flush) - def remove_tags_from_list(self, user, item, tag_to_remove_list): + def remove_tags_from_list(self, user, item, tag_to_remove_list, flush=True): tag_to_remove_set = set(tag_to_remove_list) tags_set = {_.strip() for _ in self.get_tags_str(item.tags).split(',')} if item.tags: tags_set -= tag_to_remove_set - return self.set_tags_from_list(user, item, tags_set) + return self.set_tags_from_list(user, item, tags_set, flush=flush) - def set_tags_from_list(self, user, item, new_tags_list): + def set_tags_from_list(self, user, item, new_tags_list, flush=True): # precondition: item is already security checked against user # precondition: incoming tags is a list of sanitized/formatted strings self.delete_item_tags(user, item) new_tags_str = ','.join(new_tags_list) - self.apply_item_tags(user, item, unicodify(new_tags_str, 'utf-8')) - self.sa_session.flush() + self.apply_item_tags(user, item, unicodify(new_tags_str, 'utf-8'), flush=flush) + if flush: + self.sa_session.flush() return item.tags def get_tag_assoc_class(self, item_class): @@ -146,7 +151,7 @@ class TagHandler(object): return True return False - def apply_item_tag(self, user, item, name, value=None): + def apply_item_tag(self, user, item, name, value=None, flush=True): # Use lowercase name for searching/creating tag. lc_name = name.lower() # Get or create item-tag association. @@ -174,17 +179,17 @@ class TagHandler(object): item_tag_assoc.user_tname = name item_tag_assoc.user_value = value item_tag_assoc.value = lc_value - # Need to flush to get an ID. We need an ID to apply multiple tags with the same tname to an object. - self.sa_session.flush() + if flush: + self.sa_session.flush() return item_tag_assoc - def apply_item_tags(self, user, item, tags_str): + def apply_item_tags(self, user, item, tags_str, flush=True): """Apply tags to an item.""" # Parse tags. parsed_tags = self.parse_tags(tags_str) # Apply each tag. for name, value in parsed_tags: - self.apply_item_tag(user, item, name, value) + self.apply_item_tag(user, item, name, value, flush=flush) def get_tags_str(self, tags): """Build a string from an item's tags.""" @@ -218,9 +223,9 @@ class TagHandler(object): for sub_tag in tag_hierarchy: # Get or create subtag. tag_name = tag_prefix + self._scrub_tag_name(sub_tag) - tag = self.sa_session.query(galaxy.model.Tag).filter_by(name=tag_name).first() + tag = self._get_tag(tag_name) if not tag: - tag = galaxy.model.Tag(type=0, name=tag_name) + tag = self._create_tag_instance(tag_name) # Set tag parent. tag.parent = parent_tag # Update parent and tag prefix. @@ -228,6 +233,12 @@ class TagHandler(object): tag_prefix = tag.name + self.hierarchy_separator return tag + def _get_tag(self, tag_name): + return self.sa_session.query(galaxy.model.Tag).filter_by(name=tag_name).first() + + def _create_tag_instance(self, tag_name): + return galaxy.model.Tag(type=0, name=tag_name) + def _get_or_create_tag(self, tag_str): """Get or create a Tag object from a tag string.""" # Scrub tag; if tag is None after being scrubbed, return None. @@ -365,6 +376,24 @@ class GalaxyTagHandler(TagHandler): model.VisualizationTagAssociation.table.c.visualization_id) +class GalaxyTagHandlerSession(GalaxyTagHandler): + """Like GalaxyTagHandler, but avoids one flush per created tag.""" + def __init__(self, sa_session): + super(GalaxyTagHandler, self).__init__(sa_session) + self.created_tags = {} + + def _get_tag(self, tag_name): + """Get tag from cache or database.""" + # Avoids creating multiple new tags with the same tag_name, which violates unique key constraint + return self.created_tags.get(tag_name) or super(GalaxyTagHandler, self)._get_tag(tag_name) + + def _create_tag_instance(self, tag_name): + """Create tag and and store in cache.""" + tag = super(GalaxyTagHandler)._create_tag_instance(tag_name) + self.created_tags[tag_name] = tag + return tag + + class CommunityTagHandler(TagHandler): def __init__(self, sa_session): TagHandler.__init__(self, sa_session) From f46bed896a51814f8c46c60118e3ee1d5b23c2cd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 27 Jun 2020 12:15:36 +0200 Subject: [PATCH 024/188] Avoid mutable default args in create_dataset --- lib/galaxy/model/store/discover.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 36ee7428b8e..f8c88e70d07 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -54,12 +54,16 @@ class ModelPersistenceContext(object): primary_data=None, init_from=None, dataset_attributes=None, - tag_list=[], - sources=[], - hashes=[], + tag_list=None, + sources=None, + hashes=None, created_from_basename=None, final_job_state='ok', ): + tag_list = tag_list or [] + sources = sources or [] + hashes = hashes or [] + sa_session = self.sa_session # You can initialize a dataset or initialize from a dataset but not both. From e999a601e66f405e56a9c566c2e89ffb40649eae Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 27 Jun 2020 23:32:15 +0200 Subject: [PATCH 025/188] Run through GalaxyTagHander.__init__ --- lib/galaxy/model/tags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 917d81d2abd..14785971ba8 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -379,7 +379,7 @@ class GalaxyTagHandler(TagHandler): class GalaxyTagHandlerSession(GalaxyTagHandler): """Like GalaxyTagHandler, but avoids one flush per created tag.""" def __init__(self, sa_session): - super(GalaxyTagHandler, self).__init__(sa_session) + GalaxyTagHandler.__init__(self, sa_session) self.created_tags = {} def _get_tag(self, tag_name): @@ -389,7 +389,7 @@ class GalaxyTagHandlerSession(GalaxyTagHandler): def _create_tag_instance(self, tag_name): """Create tag and and store in cache.""" - tag = super(GalaxyTagHandler)._create_tag_instance(tag_name) + tag = super(GalaxyTagHandlerSession, self)._create_tag_instance(tag_name) self.created_tags[tag_name] = tag return tag From dda22ea027ba8c40d52237019ea84407540622b6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 27 Jun 2020 23:35:38 +0200 Subject: [PATCH 026/188] Use super() to recurse into TagHandler __init__ methods And why not use the simpler py3 shortcut for `super(subclass, self)`. --- lib/galaxy/model/tags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 14785971ba8..f5792040433 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -379,7 +379,7 @@ class GalaxyTagHandler(TagHandler): class GalaxyTagHandlerSession(GalaxyTagHandler): """Like GalaxyTagHandler, but avoids one flush per created tag.""" def __init__(self, sa_session): - GalaxyTagHandler.__init__(self, sa_session) + super().__init__(sa_session) self.created_tags = {} def _get_tag(self, tag_name): @@ -389,7 +389,7 @@ class GalaxyTagHandlerSession(GalaxyTagHandler): def _create_tag_instance(self, tag_name): """Create tag and and store in cache.""" - tag = super(GalaxyTagHandlerSession, self)._create_tag_instance(tag_name) + tag = super()._create_tag_instance(tag_name) self.created_tags[tag_name] = tag return tag From cbc471b09f030148e104ec8e398d3ecd3fde4361 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 27 Jun 2020 23:43:22 +0200 Subject: [PATCH 027/188] Avoid unit test failures --- lib/galaxy/model/store/discover.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index f8c88e70d07..8bde97970d3 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -276,9 +276,14 @@ class ModelPersistenceContext(object): self.flush() def add_tags_to_datasets(self, datasets, tag_lists): - tag_session = self.tag_handler.create_tag_handler_session() - for dataset, tags in zip(datasets, tag_lists): - tag_session.add_tags_from_list(self.job.user, dataset, tags, flush=False) + if any(tag_lists): + # This works around SessionlessModelPersistenceContext not implementing a tag handler ... + # that's not better or worse than what we previously did in create_datasets + # TDOD: implement that or figure out why it is not implemented and find a better solution. + # Could it be that SessionlessModelPersistenceContext doesn't support tags? + tag_session = self.tag_handler.create_tag_handler_session() + for dataset, tags in zip(datasets, tag_lists): + tag_session.add_tags_from_list(self.job.user, dataset, tags, flush=False) @abc.abstractproperty def tag_handler(self): From dfee19b328fcd91c78183ec8fed6ddec5e19a68f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 28 Jun 2020 18:57:54 +0200 Subject: [PATCH 028/188] Add unit test for job_context.populate_collection_elements --- test/unit/jobs/test_job_context.py | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 test/unit/jobs/test_job_context.py diff --git a/test/unit/jobs/test_job_context.py b/test/unit/jobs/test_job_context.py new file mode 100644 index 00000000000..c6f2a9b7f44 --- /dev/null +++ b/test/unit/jobs/test_job_context.py @@ -0,0 +1,78 @@ +import os +import tempfile + +from galaxy import model +from galaxy.job_execution.output_collect import dataset_collector, JobContext, MetadataSourceProvider +from galaxy.model.dataset_collections import builder +from galaxy.tool_util.parser.output_collection_def import FilePatternDatasetCollectionDescription + + +from ..tools.test_history_imp_exp import _create_datasets, _mock_app, Dummy + + +class PermissionProvider: + + def __init__(self): + self.permissions = [] + + def set_default_hda_permissions(self, primary_data): + pass + + def copy_dataset_permissions(self, init_from, primary_data): + pass + +class MetadataSourceProvider: + + def get_metadata_source(self, input_name): + return None + +class Tool: + + def __init__(self, app): + self.app = app + self.sa_session = app.model.context + + +def setup_data(job_working_directory): + for i in range(10): + with open(os.path.join(job_working_directory, 'datasets_{}.txt'.format(i)), 'w') as out: + out.write(str(i)) + + +def test_job_context_discover_outputs(): + + app = _mock_app() + sa_session = app.model.context + + u = model.User(email="collection@example.com", password="password") + h = model.History(name="Test History", user=u) + + tool = Tool(app) + tool_provided_metadata = None + job = model.Job() + job.history = h + sa_session.add(job) + job_working_directory = tempfile.mkdtemp() + setup_data(job_working_directory) + permission_provider = PermissionProvider() + metadata_source_provider = MetadataSourceProvider() + object_store = app.object_store + input_dbkey = '?' + final_job_state = 'ok' + collection_description = FilePatternDatasetCollectionDescription(pattern="__name__") + collection = model.DatasetCollection(collection_type='list') + + job_context = JobContext(tool, tool_provided_metadata, job, job_working_directory, permission_provider, metadata_source_provider, input_dbkey, object_store, final_job_state) + collection_builder = builder.BoundCollectionBuilder(collection) + dataset_collectors = [dataset_collector(collection_description)] + output_name = 'output' + filenames = job_context.find_files(output_name, collection, dataset_collectors) + assert len(filenames) == 10 + job_context.populate_collection_elements( + collection, + collection_builder, + filenames, + name=output_name, + metadata_source_name='', + final_job_state=job_context.final_job_state, + ) From 15942baa2bb9e9809185ac75f49c686bdde32af9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 28 Jun 2020 18:59:04 +0200 Subject: [PATCH 029/188] Move object_store interaction out of create_dataset, avoids having to have an object_id --- lib/galaxy/model/store/discover.py | 31 +++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 8bde97970d3..aec92151de5 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -46,7 +46,7 @@ class ModelPersistenceContext(object): visible, dbkey, name, - filename, + filename=None, metadata_source_name=None, info=None, library_folder=None, @@ -127,21 +127,28 @@ class ModelPersistenceContext(object): if created_from_basename is not None: primary_data.created_from_basename = created_from_basename + has_flushed = False if tag_list: # If we have a tag we need a primary id, so need to flush here # TODO: eliminate creating tag associations within create dataset # We can do this incrementally by not passing in a tag list. self.flush() + has_flushed = True self.tag_handler.add_tags_from_list(self.job.user, primary_data, tag_list) # Move data from temp location to dataset location - if not link_data: - self.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True) - else: - primary_data.link_to(filename) + if filename: + # TODO: eliminate this, should happen outside of create_dataset so that we don't need to flush + if not has_flushed: + self.flush() + has_flushed = True + if not link_data: + self.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True) + else: + primary_data.link_to(filename) + # We are sure there are no extra files, so optimize things that follow by settting total size also. + primary_data.set_size(no_extra_files=True) - # We are sure there are no extra files, so optimize things that follow by settting total size also. - primary_data.set_size(no_extra_files=True) # If match specified a name use otherwise generate one from # designation. primary_data.name = name @@ -202,7 +209,7 @@ class ModelPersistenceContext(object): if name is None: name = "unnamed output" - element_datasets = {'element_identifiers': [], 'datasets': [], 'tag_lists': []} + element_datasets = {'element_identifiers': [], 'datasets': [], 'tag_lists': [], 'paths': []} for filename, discovered_file in filenames.items(): create_dataset_timer = ExecutionTimer() fields_match = discovered_file.match @@ -232,7 +239,6 @@ class ModelPersistenceContext(object): visible=visible, dbkey=dbkey, name=dataset_name, - filename=filename, metadata_source_name=metadata_source_name, link_data=link_data, sources=sources, @@ -251,6 +257,7 @@ class ModelPersistenceContext(object): element_datasets['element_identifiers'].append(element_identifiers) element_datasets['datasets'].append(dataset) element_datasets['tag_lists'].append(discovered_file.match.tag_list) + element_datasets['paths'].append(filename) add_datasets_timer = ExecutionTimer() self.add_datasets_to_history(element_datasets['datasets']) @@ -274,6 +281,7 @@ class ModelPersistenceContext(object): self.add_output_dataset_association(association_name, dataset) self.flush() + self.update_object_store_with_datasets(datasets=element_datasets['datasets'], paths=element_datasets['paths']) def add_tags_to_datasets(self, datasets, tag_lists): if any(tag_lists): @@ -285,6 +293,11 @@ class ModelPersistenceContext(object): for dataset, tags in zip(datasets, tag_lists): tag_session.add_tags_from_list(self.job.user, dataset, tags, flush=False) + def update_object_store_with_datasets(self, datasets, paths): + for dataset, path in zip(datasets, paths): + self.object_store.update_from_file(dataset, file_name=path, create=True) + dataset.set_size(no_extra_files=True) + @abc.abstractproperty def tag_handler(self): """Return a galaxy.model.tags.TagHandler-like object for persisting tags.""" From beaf2f91a37b6cd6b3dcf1c320fa33ab0bb80609 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 28 Jun 2020 22:29:37 +0200 Subject: [PATCH 030/188] Assert that we only flush once --- .../dependencies/pipfiles/default/Pipfile | 1 + test/unit/jobs/test_job_context.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 7bafabde224..b50b6564858 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -17,6 +17,7 @@ pygithub = "*" pytest = "*" pytest-cov = "*" pytest-html = "*" +pytest-mock = "*" pytest-pythonpath = "*" pytest-postgresql = "*" recommonmark = "*" diff --git a/test/unit/jobs/test_job_context.py b/test/unit/jobs/test_job_context.py index c6f2a9b7f44..11137ee277e 100644 --- a/test/unit/jobs/test_job_context.py +++ b/test/unit/jobs/test_job_context.py @@ -2,12 +2,13 @@ import os import tempfile from galaxy import model -from galaxy.job_execution.output_collect import dataset_collector, JobContext, MetadataSourceProvider +from galaxy.job_execution.output_collect import ( + dataset_collector, + JobContext, +) from galaxy.model.dataset_collections import builder from galaxy.tool_util.parser.output_collection_def import FilePatternDatasetCollectionDescription - - -from ..tools.test_history_imp_exp import _create_datasets, _mock_app, Dummy +from ..tools.test_history_imp_exp import _mock_app class PermissionProvider: @@ -21,11 +22,13 @@ class PermissionProvider: def copy_dataset_permissions(self, init_from, primary_data): pass + class MetadataSourceProvider: def get_metadata_source(self, input_name): return None + class Tool: def __init__(self, app): @@ -39,10 +42,10 @@ def setup_data(job_working_directory): out.write(str(i)) -def test_job_context_discover_outputs(): - +def test_job_context_discover_outputs_flushes_once(mocker): app = _mock_app() sa_session = app.model.context + # mocker is a pytest-mock fixture u = model.User(email="collection@example.com", password="password") h = model.History(name="Test History", user=u) @@ -68,6 +71,7 @@ def test_job_context_discover_outputs(): output_name = 'output' filenames = job_context.find_files(output_name, collection, dataset_collectors) assert len(filenames) == 10 + spy = mocker.spy(sa_session, 'flush') job_context.populate_collection_elements( collection, collection_builder, @@ -76,3 +80,4 @@ def test_job_context_discover_outputs(): metadata_source_name='', final_job_state=job_context.final_job_state, ) + assert spy.call_count == 1 From 14e52841c2f181519cd5f22b73aee6f7cc2ab387 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 28 Jun 2020 22:51:23 +0200 Subject: [PATCH 031/188] Build collection and assert that file size has been set --- .../pipfiles/default/pinned-dev-requirements.txt | 1 + test/unit/jobs/test_job_context.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index e738292b81b..b66360f78b3 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -37,6 +37,7 @@ pyparsing==2.4.7 pytest-cov==2.10.0 pytest-html==1.22.1 pytest-metadata==1.8.0 +pytest-mock==3.1.1 pytest-postgresql==2.3.0 pytest-pythonpath==0.7.3 pytest==5.4.3 diff --git a/test/unit/jobs/test_job_context.py b/test/unit/jobs/test_job_context.py index 11137ee277e..e9a54a04a47 100644 --- a/test/unit/jobs/test_job_context.py +++ b/test/unit/jobs/test_job_context.py @@ -63,8 +63,8 @@ def test_job_context_discover_outputs_flushes_once(mocker): input_dbkey = '?' final_job_state = 'ok' collection_description = FilePatternDatasetCollectionDescription(pattern="__name__") - collection = model.DatasetCollection(collection_type='list') - + collection = model.DatasetCollection(collection_type='list', populated=False) + sa_session.add(collection) job_context = JobContext(tool, tool_provided_metadata, job, job_working_directory, permission_provider, metadata_source_provider, input_dbkey, object_store, final_job_state) collection_builder = builder.BoundCollectionBuilder(collection) dataset_collectors = [dataset_collector(collection_description)] @@ -80,4 +80,7 @@ def test_job_context_discover_outputs_flushes_once(mocker): metadata_source_name='', final_job_state=job_context.final_job_state, ) + collection_builder.populate() assert spy.call_count == 1 + assert len(collection.dataset_instances) == 10 + assert collection.dataset_instances[0].dataset.file_size == 1 From cf7bd9195cb29134d1fe22ddf4f64da67110c6d9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 29 Jun 2020 10:29:06 +0200 Subject: [PATCH 032/188] Set metadata after updating the object store --- lib/galaxy/model/store/discover.py | 68 +++++++++++++++++------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index aec92151de5..36f9fb13766 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -63,6 +63,7 @@ class ModelPersistenceContext(object): tag_list = tag_list or [] sources = sources or [] hashes = hashes or [] + dataset_attributes = dataset_attributes or {} sa_session = self.sa_session @@ -168,38 +169,44 @@ class ModelPersistenceContext(object): if info is not None: primary_data.info = info - # add tool/metadata provided information - dataset_attributes = dataset_attributes or {} - if dataset_attributes: - # TODO: discover_files should produce a match that encorporates this - - # would simplify ToolProvidedMetadata interface and eliminate this - # crap path. - dataset_att_by_name = dict(ext='extension') - for att_set in ['name', 'info', 'ext', 'dbkey']: - dataset_att_name = dataset_att_by_name.get(att_set, att_set) - setattr(primary_data, dataset_att_name, dataset_attributes.get(att_set, getattr(primary_data, dataset_att_name))) - - try: - metadata_dict = dataset_attributes.get('metadata', None) - if metadata_dict: - if "dbkey" in dataset_attributes: - metadata_dict["dbkey"] = dataset_attributes["dbkey"] - # branch tested with tool_provided_metadata_3 / tool_provided_metadata_10 - primary_data.metadata.from_JSON_dict(json_dict=metadata_dict) - else: - primary_data.set_meta() - except Exception: - if primary_data.state == galaxy.model.HistoryDatasetAssociation.states.OK: - primary_data.state = galaxy.model.HistoryDatasetAssociation.states.FAILED_METADATA - log.exception("Exception occured while setting metdata") - - try: - primary_data.set_peek() - except Exception: - log.exception("Exception occured while setting dataset peek") + if filename: + self.set_datasets_metadata(datasets=[primary_data], datasets_attributes=[dataset_attributes]) return primary_data + @staticmethod + def set_datasets_metadata(datasets, datasets_attributes=None): + datasets_attributes = datasets_attributes or [{} for _ in datasets] + for primary_data, dataset_attributes in zip(datasets, datasets_attributes): + # add tool/metadata provided information + if dataset_attributes: + # TODO: discover_files should produce a match that encorporates this - + # would simplify ToolProvidedMetadata interface and eliminate this + # crap path. + dataset_att_by_name = dict(ext='extension') + for att_set in ['name', 'info', 'ext', 'dbkey']: + dataset_att_name = dataset_att_by_name.get(att_set, att_set) + setattr(primary_data, dataset_att_name, dataset_attributes.get(att_set, getattr(primary_data, dataset_att_name))) + + try: + metadata_dict = dataset_attributes.get('metadata', None) + if metadata_dict: + if "dbkey" in dataset_attributes: + metadata_dict["dbkey"] = dataset_attributes["dbkey"] + # branch tested with tool_provided_metadata_3 / tool_provided_metadata_10 + primary_data.metadata.from_JSON_dict(json_dict=metadata_dict) + else: + primary_data.set_meta() + except Exception: + if primary_data.state == galaxy.model.HistoryDatasetAssociation.states.OK: + primary_data.state = galaxy.model.HistoryDatasetAssociation.states.FAILED_METADATA + log.exception("Exception occured while setting metdata") + + try: + primary_data.set_peek() + except Exception: + log.exception("Exception occured while setting dataset peek") + def populate_collection_elements(self, collection, root_collection_builder, filenames, name=None, metadata_source_name=None, final_job_state='ok'): # TODO: allow configurable sorting. # @@ -282,6 +289,7 @@ class ModelPersistenceContext(object): self.flush() self.update_object_store_with_datasets(datasets=element_datasets['datasets'], paths=element_datasets['paths']) + self.set_datasets_metadata(datasets=element_datasets['datasets']) def add_tags_to_datasets(self, datasets, tag_lists): if any(tag_lists): @@ -295,7 +303,7 @@ class ModelPersistenceContext(object): def update_object_store_with_datasets(self, datasets, paths): for dataset, path in zip(datasets, paths): - self.object_store.update_from_file(dataset, file_name=path, create=True) + self.object_store.update_from_file(dataset.dataset, file_name=path, create=True) dataset.set_size(no_extra_files=True) @abc.abstractproperty From b195db877890cbe670b528896440401bab5df2bf Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Mon, 29 Jun 2020 09:23:19 -0400 Subject: [PATCH 033/188] Add unit test to regex validator --- lib/galaxy/tools/parameters/validation.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index f75a31bc6be..f31a261d74f 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -66,6 +66,11 @@ class RegexValidator(Validator): Traceback (most recent call last): ... ValueError: Not gonna happen + >>> t = p.validate(["Foo", "foo"]) + >>> t = p.validate(["Foo", "Fop"]) + Traceback (most recent call last): + ... + ValueError: Not gonna happen """ @classmethod From e2dee6fa22b370d4a4025fddf42641cf0da66477 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Tue, 12 May 2020 21:41:38 +0200 Subject: [PATCH 034/188] fix registering 3+ aguments in workflow report --- .../scripts/components/Markdown/Markdown.vue | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/client/galaxy/scripts/components/Markdown/Markdown.vue b/client/galaxy/scripts/components/Markdown/Markdown.vue index 9ac141bcaea..67b10584a64 100644 --- a/client/galaxy/scripts/components/Markdown/Markdown.vue +++ b/client/galaxy/scripts/components/Markdown/Markdown.vue @@ -22,7 +22,7 @@ import HDCAListItemEdit from "mvc/history/hdca-li-edit"; import HDCAListItem from "mvc/history/hdca-li"; const FUNCTION_VALUE_REGEX = `\\s*(?:[\\w_\\-]+|\\"[^\\"]+\\"|\\'[^\\']+\\')\\s*`; const FUNCTION_CALL = `\\s*\\w+\\s*=` + FUNCTION_VALUE_REGEX; -const FUNCTION_CALL_LINE = `\\s*(\\w+)\\s*\\((` + FUNCTION_CALL + `)(,` + FUNCTION_CALL + `)*\\)\\s*`; +const FUNCTION_CALL_LINE = `\\s*(\\w+)\\s*\\((${FUNCTION_CALL})(,${FUNCTION_CALL})*\\)\\s*`; const FUNCTION_CALL_LINE_TEMPLATE = new RegExp(FUNCTION_CALL_LINE, "m"); const md = MarkdownIt(); @@ -61,6 +61,10 @@ const RENDER_FUNCTIONS = { `; }, + history_dataset_composite: (action, args, content) => { + const history_dataset_id = args.history_dataset_id; + return `
`; @@ -123,21 +127,26 @@ md.renderer.rules.fence = function (tokens, idx, options, env, slf) { const token = tokens[idx]; const info = token.info ? token.info.trim() : ""; const content = token.content; + if (info == "galaxy") { - const arr = FUNCTION_CALL_LINE_TEMPLATE.exec(content); + const galaxy_function = FUNCTION_CALL_LINE_TEMPLATE.exec(content); const args = {}; - const action = arr[1]; - for (let i = 2; i < arr.length; i++) { - if (arr[i] === undefined) continue; - const arguments_str = arr[i].replace(/,/g, "").trim(); + const function_name = galaxy_function[1]; + // we need [... ] to return empty string, if regex doesn't match + const function_arguments = [...content.matchAll(new RegExp(FUNCTION_CALL, "g"))] + + for (let i = 0; i < function_arguments.length; i++) { + if (function_arguments[i] === undefined) continue; + const arguments_str = function_arguments[i].toString().replace(/,/g, "").trim(); if (arguments_str) { const [key, val] = arguments_str.split("="); args[key.trim()] = val.replace(/['"]+/g, "").trim(); } } - return RENDER_FUNCTIONS[action](action, args, content); + + return RENDER_FUNCTIONS[function_name](function_name, args, content); } else { return default_fence(tokens, idx, options, env, slf); } From 1b2906bb94651b6798ad2a06216aa46ffb99a711 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Wed, 13 May 2020 22:21:39 +0200 Subject: [PATCH 035/188] download invidual files from composite dataset --- client/galaxy/scripts/bundleEntries.js | 1 + .../CompositeDataset/CompositeDataset.vue | 33 +++++++ .../CompositeDataset/CompositeDatasetFile.vue | 91 +++++++++++++++++++ .../components/CompositeDataset/index.js | 3 + .../components/CompositeDataset/mount.js | 16 ++++ .../components/CompositeDataset/services.js | 37 ++++++++ .../scripts/components/Markdown/Markdown.vue | 8 +- lib/galaxy/managers/markdown_parse.py | 1 + lib/galaxy/managers/markdown_util.py | 4 + 9 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue create mode 100644 client/galaxy/scripts/components/CompositeDataset/CompositeDatasetFile.vue create mode 100644 client/galaxy/scripts/components/CompositeDataset/index.js create mode 100644 client/galaxy/scripts/components/CompositeDataset/mount.js create mode 100644 client/galaxy/scripts/components/CompositeDataset/services.js diff --git a/client/galaxy/scripts/bundleEntries.js b/client/galaxy/scripts/bundleEntries.js index 8e6efed234d..d66b55b0c90 100644 --- a/client/galaxy/scripts/bundleEntries.js +++ b/client/galaxy/scripts/bundleEntries.js @@ -96,6 +96,7 @@ export { mountJobParameters } from "components/JobParameters"; export { mountWorkflowEditor } from "components/Workflow/Editor/mount"; export { mountPageDisplay } from "components/PageDisplay"; export { mountDestinationParams } from "components/JobDestinationParams"; +export { mountCompositeDataset } from "components/CompositeDataset"; // Used in common.mako export { default as store } from "storemodern"; diff --git a/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue b/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue new file mode 100644 index 00000000000..a47e2ebc39a --- /dev/null +++ b/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue @@ -0,0 +1,33 @@ + + + diff --git a/client/galaxy/scripts/components/CompositeDataset/CompositeDatasetFile.vue b/client/galaxy/scripts/components/CompositeDataset/CompositeDatasetFile.vue new file mode 100644 index 00000000000..6fc7f08fba8 --- /dev/null +++ b/client/galaxy/scripts/components/CompositeDataset/CompositeDatasetFile.vue @@ -0,0 +1,91 @@ + + + + + diff --git a/client/galaxy/scripts/components/CompositeDataset/index.js b/client/galaxy/scripts/components/CompositeDataset/index.js new file mode 100644 index 00000000000..03079297e13 --- /dev/null +++ b/client/galaxy/scripts/components/CompositeDataset/index.js @@ -0,0 +1,3 @@ +export { default as CompositeDataset } from "./CompositeDataset"; + +export { mountCompositeDataset } from "./mount"; diff --git a/client/galaxy/scripts/components/CompositeDataset/mount.js b/client/galaxy/scripts/components/CompositeDataset/mount.js new file mode 100644 index 00000000000..0e1fb1e24c9 --- /dev/null +++ b/client/galaxy/scripts/components/CompositeDataset/mount.js @@ -0,0 +1,16 @@ +/** + * Endpoint for mounting job metrics from non-Vue environment. + */ +import $ from "jquery"; +import CompositeDataset from "./CompositeDataset.vue"; +import { mountVueComponent } from "utils/mountVueComponent"; + +export const mountCompositeDataset = (propsData = {}) => { + + $(".composite-dataset").each((index, el) => { + propsData.history_dataset_id = $(el).attr("history_dataset_id"); + propsData.path = $(el).attr("path"); + propsData.label = $(el).attr("label"); + mountVueComponent(CompositeDataset)(propsData, el); + }); +}; diff --git a/client/galaxy/scripts/components/CompositeDataset/services.js b/client/galaxy/scripts/components/CompositeDataset/services.js new file mode 100644 index 00000000000..923dffb6eed --- /dev/null +++ b/client/galaxy/scripts/components/CompositeDataset/services.js @@ -0,0 +1,37 @@ +import axios from "axios"; +import { rethrowSimple } from "utils/simple-error"; +import { getAppRoot } from "onload/loadConfig"; + +export class Services { + constructor(options = {}) { + this.root = options.root || getAppRoot(); + } + + async getCompositeDatasetContentFiles(id) { + const url = `${this.root}api/histories/${id}/contents/${id}/extra_files`; + try { + const response = await axios.get(url); + return response.data; + } catch (e) { + rethrowSimple(e); + } + } + async getCompositeDatasetDirectory(url) { + try { + const response = await axios.get(url); + return response.data; + } catch (e) { + rethrowSimple(e); + } + } + async getCompositeDatasetInfo(id) { + const url = `${this.root}api/histories/${id}/contents/${id}`; + try { + const response = await axios.get(url); + return response.data; + } catch (e) { + rethrowSimple(e); + } + } + +} diff --git a/client/galaxy/scripts/components/Markdown/Markdown.vue b/client/galaxy/scripts/components/Markdown/Markdown.vue index 67b10584a64..197afa3bc16 100644 --- a/client/galaxy/scripts/components/Markdown/Markdown.vue +++ b/client/galaxy/scripts/components/Markdown/Markdown.vue @@ -14,6 +14,7 @@ import { getGalaxyInstance } from "app"; import { render_embedded_items } from "mvc/embedded-objects"; import { mountJobMetrics } from "components/JobMetrics"; import { mountJobParameters } from "components/JobParameters"; +import { mountCompositeDataset } from "components/CompositeDataset"; import MarkdownIt from "markdown-it"; import JOB_STATES_MODEL from "mvc/history/job-states-model"; @@ -63,7 +64,10 @@ const RENDER_FUNCTIONS = { }, history_dataset_composite: (action, args, content) => { const history_dataset_id = args.history_dataset_id; - return `
`; }, history_dataset_collection_display: (action, args, content) => { const history_dataset_collection_id = args.history_dataset_collection_id; @@ -202,6 +206,8 @@ export default { render_embedded_items(); mountJobMetrics({ includeTitle: false }); mountJobParameters({ includeTitle: false }); + mountCompositeDataset(); + $("span.render-name").each((i, el) => { const historyDatasetId = $(el).attr("history_dataset_id"); if (historyDatasetId) { diff --git a/lib/galaxy/managers/markdown_parse.py b/lib/galaxy/managers/markdown_parse.py index deb3ebad659..efbc38dd7e5 100644 --- a/lib/galaxy/managers/markdown_parse.py +++ b/lib/galaxy/managers/markdown_parse.py @@ -20,6 +20,7 @@ GALAXY_FLAVORED_MARKDOWN_CONTAINERS = [ "history_dataset_as_image", "history_dataset_peek", "history_dataset_info", + "history_dataset_composite", "workflow_display", "job_metrics", "job_parameters", diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 084afe9975f..c127b488595 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -98,6 +98,10 @@ class GalaxyInternalMarkdownDirectiveHandler(object): assert object_id is not None hda = hda_manager.get_accessible(object_id, trans.user) rval = self.handle_dataset_display(line, hda) + elif container == "history_dataset_composite": + assert object_id is not None + hda = hda_manager.get_accessible(object_id, trans.user) + rval = self.handle_dataset_display(line, hda) elif container == "history_dataset_embedded": assert object_id is not None hda = hda_manager.get_accessible(object_id, trans.user) From c42d8c21b90741bf4146e186dcf4a1ed08389168 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 14 May 2020 17:48:24 +0200 Subject: [PATCH 036/188] implement comsosite dataset image render and downloading whole dataset --- .../CompositeDataset/CompositeDataset.vue | 98 +++++++++++++++++-- .../CompositeDataset/CompositeDatasetFile.vue | 91 ----------------- .../components/CompositeDataset/mount.js | 1 + .../scripts/components/Markdown/Markdown.vue | 3 +- 4 files changed, 91 insertions(+), 102 deletions(-) delete mode 100644 client/galaxy/scripts/components/CompositeDataset/CompositeDatasetFile.vue diff --git a/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue b/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue index a47e2ebc39a..9a69483c28e 100644 --- a/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue +++ b/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue @@ -1,16 +1,36 @@ + + diff --git a/client/galaxy/scripts/components/CompositeDataset/CompositeDatasetFile.vue b/client/galaxy/scripts/components/CompositeDataset/CompositeDatasetFile.vue deleted file mode 100644 index 6fc7f08fba8..00000000000 --- a/client/galaxy/scripts/components/CompositeDataset/CompositeDatasetFile.vue +++ /dev/null @@ -1,91 +0,0 @@ - - - - - diff --git a/client/galaxy/scripts/components/CompositeDataset/mount.js b/client/galaxy/scripts/components/CompositeDataset/mount.js index 0e1fb1e24c9..85b1f2d9df3 100644 --- a/client/galaxy/scripts/components/CompositeDataset/mount.js +++ b/client/galaxy/scripts/components/CompositeDataset/mount.js @@ -11,6 +11,7 @@ export const mountCompositeDataset = (propsData = {}) => { propsData.history_dataset_id = $(el).attr("history_dataset_id"); propsData.path = $(el).attr("path"); propsData.label = $(el).attr("label"); + propsData.image = $(el).attr("image"); mountVueComponent(CompositeDataset)(propsData, el); }); }; diff --git a/client/galaxy/scripts/components/Markdown/Markdown.vue b/client/galaxy/scripts/components/Markdown/Markdown.vue index 197afa3bc16..2f6242bb9f4 100644 --- a/client/galaxy/scripts/components/Markdown/Markdown.vue +++ b/client/galaxy/scripts/components/Markdown/Markdown.vue @@ -66,8 +66,9 @@ const RENDER_FUNCTIONS = { const history_dataset_id = args.history_dataset_id; const path = args.path; const label = args.label; + const image = args.image; - return `
`; + return `
`; }, history_dataset_collection_display: (action, args, content) => { const history_dataset_collection_id = args.history_dataset_collection_id; From a652a041276a40c0a3a802aa6848e9dd488b2b59 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 14 May 2020 19:36:00 +0200 Subject: [PATCH 037/188] fix v-if and run prettier --- .../scripts/components/CompositeDataset/CompositeDataset.vue | 4 ++-- client/galaxy/scripts/components/CompositeDataset/mount.js | 1 - client/galaxy/scripts/components/CompositeDataset/services.js | 1 - client/galaxy/scripts/components/Markdown/Markdown.vue | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue b/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue index 9a69483c28e..ca051a0d0f2 100644 --- a/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue +++ b/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue @@ -1,10 +1,10 @@ diff --git a/client/galaxy/scripts/components/CompositeDataset/mount.js b/client/galaxy/scripts/components/CompositeDataset/mount.js index 85b1f2d9df3..6d826625c16 100644 --- a/client/galaxy/scripts/components/CompositeDataset/mount.js +++ b/client/galaxy/scripts/components/CompositeDataset/mount.js @@ -6,7 +6,6 @@ import CompositeDataset from "./CompositeDataset.vue"; import { mountVueComponent } from "utils/mountVueComponent"; export const mountCompositeDataset = (propsData = {}) => { - $(".composite-dataset").each((index, el) => { propsData.history_dataset_id = $(el).attr("history_dataset_id"); propsData.path = $(el).attr("path"); diff --git a/client/galaxy/scripts/components/CompositeDataset/services.js b/client/galaxy/scripts/components/CompositeDataset/services.js index 923dffb6eed..75e2aa710fd 100644 --- a/client/galaxy/scripts/components/CompositeDataset/services.js +++ b/client/galaxy/scripts/components/CompositeDataset/services.js @@ -33,5 +33,4 @@ export class Services { rethrowSimple(e); } } - } diff --git a/client/galaxy/scripts/components/Markdown/Markdown.vue b/client/galaxy/scripts/components/Markdown/Markdown.vue index 2f6242bb9f4..ccf59b27568 100644 --- a/client/galaxy/scripts/components/Markdown/Markdown.vue +++ b/client/galaxy/scripts/components/Markdown/Markdown.vue @@ -139,7 +139,7 @@ md.renderer.rules.fence = function (tokens, idx, options, env, slf) { const args = {}; const function_name = galaxy_function[1]; // we need [... ] to return empty string, if regex doesn't match - const function_arguments = [...content.matchAll(new RegExp(FUNCTION_CALL, "g"))] + const function_arguments = [...content.matchAll(new RegExp(FUNCTION_CALL, "g"))]; for (let i = 0; i < function_arguments.length; i++) { if (function_arguments[i] === undefined) continue; From 47a0bbd23a005f09ad45cd5162e608d4110cc65f Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 14 May 2020 19:49:23 +0200 Subject: [PATCH 038/188] fix lint in CompositeDataset --- .../CompositeDataset/CompositeDataset.vue | 161 +++++++++--------- 1 file changed, 78 insertions(+), 83 deletions(-) diff --git a/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue b/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue index ca051a0d0f2..a1e02221364 100644 --- a/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue +++ b/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue @@ -1,111 +1,106 @@ From b8301ed118806b897bd71e7454d7415963cf843a Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Mon, 18 May 2020 21:49:39 +0200 Subject: [PATCH 039/188] history_dataset_composite -> history_dataset_link --- client/galaxy/scripts/bundleEntries.js | 2 +- .../scripts/components/CompositeDataset/index.js | 3 --- .../DatasetLink.vue} | 0 client/galaxy/scripts/components/DatasetLink/index.js | 3 +++ .../{CompositeDataset => DatasetLink}/mount.js | 10 +++++----- .../{CompositeDataset => DatasetLink}/services.js | 0 client/galaxy/scripts/components/Markdown/Markdown.vue | 8 ++++---- lib/galaxy/managers/markdown_parse.py | 2 +- lib/galaxy/managers/markdown_util.py | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 client/galaxy/scripts/components/CompositeDataset/index.js rename client/galaxy/scripts/components/{CompositeDataset/CompositeDataset.vue => DatasetLink/DatasetLink.vue} (100%) create mode 100644 client/galaxy/scripts/components/DatasetLink/index.js rename client/galaxy/scripts/components/{CompositeDataset => DatasetLink}/mount.js (52%) rename client/galaxy/scripts/components/{CompositeDataset => DatasetLink}/services.js (100%) diff --git a/client/galaxy/scripts/bundleEntries.js b/client/galaxy/scripts/bundleEntries.js index d66b55b0c90..61b90d77710 100644 --- a/client/galaxy/scripts/bundleEntries.js +++ b/client/galaxy/scripts/bundleEntries.js @@ -96,7 +96,7 @@ export { mountJobParameters } from "components/JobParameters"; export { mountWorkflowEditor } from "components/Workflow/Editor/mount"; export { mountPageDisplay } from "components/PageDisplay"; export { mountDestinationParams } from "components/JobDestinationParams"; -export { mountCompositeDataset } from "components/CompositeDataset"; +export { mountDatasetLink } from "components/DatasetLink"; // Used in common.mako export { default as store } from "storemodern"; diff --git a/client/galaxy/scripts/components/CompositeDataset/index.js b/client/galaxy/scripts/components/CompositeDataset/index.js deleted file mode 100644 index 03079297e13..00000000000 --- a/client/galaxy/scripts/components/CompositeDataset/index.js +++ /dev/null @@ -1,3 +0,0 @@ -export { default as CompositeDataset } from "./CompositeDataset"; - -export { mountCompositeDataset } from "./mount"; diff --git a/client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue b/client/galaxy/scripts/components/DatasetLink/DatasetLink.vue similarity index 100% rename from client/galaxy/scripts/components/CompositeDataset/CompositeDataset.vue rename to client/galaxy/scripts/components/DatasetLink/DatasetLink.vue diff --git a/client/galaxy/scripts/components/DatasetLink/index.js b/client/galaxy/scripts/components/DatasetLink/index.js new file mode 100644 index 00000000000..54db93cb1b4 --- /dev/null +++ b/client/galaxy/scripts/components/DatasetLink/index.js @@ -0,0 +1,3 @@ +export { default as DatasetLink } from "./DatasetLink"; + +export { mountDatasetLink } from "./mount"; diff --git a/client/galaxy/scripts/components/CompositeDataset/mount.js b/client/galaxy/scripts/components/DatasetLink/mount.js similarity index 52% rename from client/galaxy/scripts/components/CompositeDataset/mount.js rename to client/galaxy/scripts/components/DatasetLink/mount.js index 6d826625c16..6259707966c 100644 --- a/client/galaxy/scripts/components/CompositeDataset/mount.js +++ b/client/galaxy/scripts/components/DatasetLink/mount.js @@ -1,16 +1,16 @@ /** - * Endpoint for mounting job metrics from non-Vue environment. + * Endpoint for mounting dataset link from non-Vue environment. */ import $ from "jquery"; -import CompositeDataset from "./CompositeDataset.vue"; +import DatasetLink from "./DatasetLink"; import { mountVueComponent } from "utils/mountVueComponent"; -export const mountCompositeDataset = (propsData = {}) => { - $(".composite-dataset").each((index, el) => { +export const mountDatasetLink = (propsData = {}) => { + $(".dataset-link").each((index, el) => { propsData.history_dataset_id = $(el).attr("history_dataset_id"); propsData.path = $(el).attr("path"); propsData.label = $(el).attr("label"); propsData.image = $(el).attr("image"); - mountVueComponent(CompositeDataset)(propsData, el); + mountVueComponent(DatasetLink)(propsData, el); }); }; diff --git a/client/galaxy/scripts/components/CompositeDataset/services.js b/client/galaxy/scripts/components/DatasetLink/services.js similarity index 100% rename from client/galaxy/scripts/components/CompositeDataset/services.js rename to client/galaxy/scripts/components/DatasetLink/services.js diff --git a/client/galaxy/scripts/components/Markdown/Markdown.vue b/client/galaxy/scripts/components/Markdown/Markdown.vue index ccf59b27568..2e39ffb3297 100644 --- a/client/galaxy/scripts/components/Markdown/Markdown.vue +++ b/client/galaxy/scripts/components/Markdown/Markdown.vue @@ -14,7 +14,7 @@ import { getGalaxyInstance } from "app"; import { render_embedded_items } from "mvc/embedded-objects"; import { mountJobMetrics } from "components/JobMetrics"; import { mountJobParameters } from "components/JobParameters"; -import { mountCompositeDataset } from "components/CompositeDataset"; +import { mountDatasetLink } from "components/DatasetLink"; import MarkdownIt from "markdown-it"; import JOB_STATES_MODEL from "mvc/history/job-states-model"; @@ -62,13 +62,13 @@ const RENDER_FUNCTIONS = { `; }, - history_dataset_composite: (action, args, content) => { + history_dataset_link: (action, args, content) => { const history_dataset_id = args.history_dataset_id; const path = args.path; const label = args.label; const image = args.image; - return `
`; + return `
`; }, history_dataset_collection_display: (action, args, content) => { const history_dataset_collection_id = args.history_dataset_collection_id; @@ -207,7 +207,7 @@ export default { render_embedded_items(); mountJobMetrics({ includeTitle: false }); mountJobParameters({ includeTitle: false }); - mountCompositeDataset(); + mountDatasetLink(); $("span.render-name").each((i, el) => { const historyDatasetId = $(el).attr("history_dataset_id"); diff --git a/lib/galaxy/managers/markdown_parse.py b/lib/galaxy/managers/markdown_parse.py index efbc38dd7e5..fed43263f77 100644 --- a/lib/galaxy/managers/markdown_parse.py +++ b/lib/galaxy/managers/markdown_parse.py @@ -20,7 +20,7 @@ GALAXY_FLAVORED_MARKDOWN_CONTAINERS = [ "history_dataset_as_image", "history_dataset_peek", "history_dataset_info", - "history_dataset_composite", + "history_dataset_link", "workflow_display", "job_metrics", "job_parameters", diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index c127b488595..25b74699738 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -98,7 +98,7 @@ class GalaxyInternalMarkdownDirectiveHandler(object): assert object_id is not None hda = hda_manager.get_accessible(object_id, trans.user) rval = self.handle_dataset_display(line, hda) - elif container == "history_dataset_composite": + elif container == "history_dataset_link": assert object_id is not None hda = hda_manager.get_accessible(object_id, trans.user) rval = self.handle_dataset_display(line, hda) From 9161b0bff7f545e261a17f0915ae28f4e071f2ac Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Mon, 25 May 2020 20:35:50 +0200 Subject: [PATCH 040/188] implement history_dataset_index --- client/galaxy/scripts/bundleEntries.js | 2 +- .../Dataset/DatasetIndex/DatasetIndex.vue | 90 +++++++++++++++++++ .../components/Dataset/DatasetIndex/index.js | 3 + .../components/Dataset/DatasetIndex/mount.js | 14 +++ .../{ => Dataset}/DatasetLink/DatasetLink.vue | 21 +++-- .../{ => Dataset}/DatasetLink/index.js | 0 .../{ => Dataset}/DatasetLink/mount.js | 0 .../scripts/components/Dataset/services.js | 30 +++++++ .../components/DatasetLink/services.js | 36 -------- .../scripts/components/Markdown/Markdown.vue | 9 +- lib/galaxy/managers/markdown_parse.py | 1 + lib/galaxy/managers/markdown_util.py | 4 + 12 files changed, 161 insertions(+), 49 deletions(-) create mode 100644 client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue create mode 100644 client/galaxy/scripts/components/Dataset/DatasetIndex/index.js create mode 100644 client/galaxy/scripts/components/Dataset/DatasetIndex/mount.js rename client/galaxy/scripts/components/{ => Dataset}/DatasetLink/DatasetLink.vue (79%) rename client/galaxy/scripts/components/{ => Dataset}/DatasetLink/index.js (100%) rename client/galaxy/scripts/components/{ => Dataset}/DatasetLink/mount.js (100%) delete mode 100644 client/galaxy/scripts/components/DatasetLink/services.js diff --git a/client/galaxy/scripts/bundleEntries.js b/client/galaxy/scripts/bundleEntries.js index 61b90d77710..6808c0a4bb7 100644 --- a/client/galaxy/scripts/bundleEntries.js +++ b/client/galaxy/scripts/bundleEntries.js @@ -96,7 +96,7 @@ export { mountJobParameters } from "components/JobParameters"; export { mountWorkflowEditor } from "components/Workflow/Editor/mount"; export { mountPageDisplay } from "components/PageDisplay"; export { mountDestinationParams } from "components/JobDestinationParams"; -export { mountDatasetLink } from "components/DatasetLink"; +export { mountDatasetLink } from "components/Dataset/DatasetLink"; // Used in common.mako export { default as store } from "storemodern"; diff --git a/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue b/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue new file mode 100644 index 00000000000..2893562cfb0 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue @@ -0,0 +1,90 @@ + + + diff --git a/client/galaxy/scripts/components/Dataset/DatasetIndex/index.js b/client/galaxy/scripts/components/Dataset/DatasetIndex/index.js new file mode 100644 index 00000000000..064c75bec23 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/DatasetIndex/index.js @@ -0,0 +1,3 @@ +export { default as DatasetIndex } from "./DatasetIndex"; + +export { mountDatasetIndex } from "./mount"; diff --git a/client/galaxy/scripts/components/Dataset/DatasetIndex/mount.js b/client/galaxy/scripts/components/Dataset/DatasetIndex/mount.js new file mode 100644 index 00000000000..820d1583038 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/DatasetIndex/mount.js @@ -0,0 +1,14 @@ +/** + * Endpoint for mounting dataset link from non-Vue environment. + */ +import $ from "jquery"; +import DatasetIndex from "./DatasetIndex"; +import { mountVueComponent } from "utils/mountVueComponent"; + +export const mountDatasetIndex = (propsData = {}) => { + $(".dataset-index").each((index, el) => { + propsData.history_dataset_id = $(el).attr("history_dataset_id"); + propsData.path = $(el).attr("path"); + mountVueComponent(DatasetIndex)(propsData, el); + }); +}; diff --git a/client/galaxy/scripts/components/DatasetLink/DatasetLink.vue b/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue similarity index 79% rename from client/galaxy/scripts/components/DatasetLink/DatasetLink.vue rename to client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue index a1e02221364..324e381a78b 100644 --- a/client/galaxy/scripts/components/DatasetLink/DatasetLink.vue +++ b/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue @@ -1,9 +1,6 @@ @@ -27,7 +24,7 @@ div.directory-content { diff --git a/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue b/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue index 2893562cfb0..efe37be8130 100644 --- a/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue +++ b/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue @@ -10,14 +10,13 @@ >
- {{ path }} {{ errorMessage }} + {{ path }} {{ errorMessage }}
diff --git a/client/galaxy/scripts/components/Dataset/DatasetLink/index.js b/client/galaxy/scripts/components/Dataset/DatasetLink/index.js deleted file mode 100644 index 54db93cb1b4..00000000000 --- a/client/galaxy/scripts/components/Dataset/DatasetLink/index.js +++ /dev/null @@ -1,3 +0,0 @@ -export { default as DatasetLink } from "./DatasetLink"; - -export { mountDatasetLink } from "./mount"; diff --git a/client/galaxy/scripts/components/Dataset/DatasetLink/mount.js b/client/galaxy/scripts/components/Dataset/DatasetLink/mount.js deleted file mode 100644 index 6259707966c..00000000000 --- a/client/galaxy/scripts/components/Dataset/DatasetLink/mount.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Endpoint for mounting dataset link from non-Vue environment. - */ -import $ from "jquery"; -import DatasetLink from "./DatasetLink"; -import { mountVueComponent } from "utils/mountVueComponent"; - -export const mountDatasetLink = (propsData = {}) => { - $(".dataset-link").each((index, el) => { - propsData.history_dataset_id = $(el).attr("history_dataset_id"); - propsData.path = $(el).attr("path"); - propsData.label = $(el).attr("label"); - propsData.image = $(el).attr("image"); - mountVueComponent(DatasetLink)(propsData, el); - }); -}; diff --git a/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js b/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js new file mode 100644 index 00000000000..d8580a666e5 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js @@ -0,0 +1,31 @@ +import { getAppRoot } from "onload/loadConfig"; +import { Services } from "components/Dataset/services"; + +export const getPathDestination = (history_dataset_id, path) => { + const services = new Services({ root: getAppRoot() }); + return services.getCompositeDatasetContentFiles(history_dataset_id).then((datasetContent) => { + const pathDestination = { datasetContent: datasetContent }; + if (datasetContent[0].class === "Directory") pathDestination.datasetRootDir = datasetContent[0].path; + else return; + + if (path === undefined || path === "undefined") { + return pathDestination; + } + + const filepath = `${pathDestination.datasetRootDir}/${path}`; + + const datasetEntry = datasetContent.find((datasetEntry) => { + return filepath === datasetEntry.path; + }); + + if (datasetEntry) { + if (datasetEntry.class === "Directory") { + pathDestination.isDirectory = true; + pathDestination.filepath = filepath; + return pathDestination; + } + pathDestination.fileLink = services.getCompositeDatasetLink(history_dataset_id, datasetEntry.path); + } + return pathDestination; + }); +}; diff --git a/client/galaxy/scripts/components/Dataset/mount.js b/client/galaxy/scripts/components/Dataset/mount.js new file mode 100644 index 00000000000..85bdddfa847 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/mount.js @@ -0,0 +1,32 @@ +/** + * Endpoint for mounting dataset link from non-Vue environment. + */ +import $ from "jquery"; +import DatasetLink from "./DatasetLink/DatasetLink"; +import { mountVueComponent } from "utils/mountVueComponent"; +import DatasetIndex from "./DatasetIndex/DatasetIndex"; +import DatasetAsImage from "./DatasetAsImage/DatasetAsImage"; + +export const mountDatasetAsImage = () => { + return mountDatasetComponent(".dataset-as-image", ["history_dataset_id", "path"], DatasetAsImage); +}; +export const mountDatasetIndex = () => { + return mountDatasetComponent(".dataset-index", ["history_dataset_id", "path"], DatasetIndex); +}; +export const mountDatasetLink = () => { + return mountDatasetComponent(".dataset-link", ["history_dataset_id", "path", "label"], DatasetLink); +}; + +export const mountDatasetComponents = () => { + const all = [mountDatasetAsImage, mountDatasetIndex, mountDatasetLink]; + all.forEach((func) => func()); +}; + +const mountDatasetComponent = (selector, props, component, propsData = {}) => { + $(selector).each((index, el) => { + props.forEach((prop) => { + propsData[prop] = $(el).attr(prop); + }); + mountVueComponent(component)(propsData, el); + }); +}; diff --git a/client/galaxy/scripts/components/Dataset/services.js b/client/galaxy/scripts/components/Dataset/services.js index 5ffa47d36e8..f276215d467 100644 --- a/client/galaxy/scripts/components/Dataset/services.js +++ b/client/galaxy/scripts/components/Dataset/services.js @@ -79,13 +79,8 @@ export class Services { } } - async getCompositeDatasetDirectory(url) { - try { - const response = await axios.get(url); - return response.data; - } catch (e) { - rethrowSimple(e); - } + getCompositeDatasetLink(history_dataset_id, path) { + return `${this.root}api/histories/${history_dataset_id}/contents/${history_dataset_id}/display?filename=${path}`; } async getCompositeDatasetInfo(id) { diff --git a/client/galaxy/scripts/components/Markdown/Markdown.vue b/client/galaxy/scripts/components/Markdown/Markdown.vue index caab9459a67..9da952c098b 100644 --- a/client/galaxy/scripts/components/Markdown/Markdown.vue +++ b/client/galaxy/scripts/components/Markdown/Markdown.vue @@ -14,8 +14,7 @@ import { getGalaxyInstance } from "app"; import { render_embedded_items } from "mvc/embedded-objects"; import { mountJobMetrics } from "components/JobMetrics"; import { mountJobParameters } from "components/JobParameters"; -import { mountDatasetLink } from "components/Dataset/DatasetLink"; -import { mountDatasetIndex } from "components/Dataset/DatasetIndex"; +import { mountDatasetComponents } from "components/Dataset/mount"; import MarkdownIt from "markdown-it"; import JOB_STATES_MODEL from "mvc/history/job-states-model"; @@ -67,9 +66,7 @@ const RENDER_FUNCTIONS = { const history_dataset_id = args.history_dataset_id; const path = args.path; const label = args.label; - const image = args.image; - - return `
`; + return `
`; }, history_dataset_index: (action, args, content) => { const history_dataset_id = args.history_dataset_id; @@ -105,7 +102,11 @@ const RENDER_FUNCTIONS = { }, history_dataset_as_image: (action, args, content) => { const history_dataset_id = args.history_dataset_id; - return ``; + const path = args.path; + if (path) { + `${getAppRoot}api/histories/${history_dataset_id}/contents/${history_dataset_id}/display?filename=${path}`; + } + return `
`; }, history_dataset_peek: (action, args, content) => { const history_dataset_id = args.history_dataset_id; @@ -213,8 +214,7 @@ export default { render_embedded_items(); mountJobMetrics({ includeTitle: false }); mountJobParameters({ includeTitle: false }); - mountDatasetLink(); - mountDatasetIndex(); + mountDatasetComponents(); $("span.render-name").each((i, el) => { const historyDatasetId = $(el).attr("history_dataset_id"); From e961a13182e69773a6c1443d6b371d1ef4d3e91c Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Tue, 26 May 2020 20:54:07 +0200 Subject: [PATCH 042/188] check if image link is not available --- .../components/Dataset/DatasetAsImage/DatasetAsImage.vue | 5 ++++- client/galaxy/scripts/components/Dataset/mount.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue b/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue index 06f56e13d2a..955ee315bd6 100644 --- a/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue +++ b/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue @@ -1,6 +1,9 @@ diff --git a/client/galaxy/scripts/components/Dataset/mount.js b/client/galaxy/scripts/components/Dataset/mount.js index 85bdddfa847..a577a4374e4 100644 --- a/client/galaxy/scripts/components/Dataset/mount.js +++ b/client/galaxy/scripts/components/Dataset/mount.js @@ -1,5 +1,5 @@ /** - * Endpoint for mounting dataset link from non-Vue environment. + * Endpoint for mounting dataset components from non-Vue environment. */ import $ from "jquery"; import DatasetLink from "./DatasetLink/DatasetLink"; From 88a0ba740993c04e4af2a727e548e44fc9cb70f8 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Wed, 27 May 2020 19:01:46 +0200 Subject: [PATCH 043/188] implement Dataset Path Destination caching store. --- .../Dataset/DatasetLink/DatasetLink.vue | 12 +++---- .../store/datasetPathDestinationStore.js | 35 +++++++++++++++++++ client/galaxy/scripts/store/index.js | 2 ++ 3 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 client/galaxy/scripts/store/datasetPathDestinationStore.js diff --git a/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue b/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue index 8ec5243dda8..15be735a64a 100644 --- a/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue +++ b/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue @@ -7,7 +7,7 @@ diff --git a/client/galaxy/scripts/store/datasetPathDestinationStore.js b/client/galaxy/scripts/store/datasetPathDestinationStore.js new file mode 100644 index 00000000000..b687a3aa1ce --- /dev/null +++ b/client/galaxy/scripts/store/datasetPathDestinationStore.js @@ -0,0 +1,35 @@ +export const state = { + datasetPathDestination: {}, +}; + +import { getPathDestination } from "components/Dataset/compositeDatasetUtils"; + +const getters = { + pathDestination: (state) => (history_dataset_id, path) => { + if (state.datasetPathDestination[history_dataset_id]) { + return state.datasetPathDestination[history_dataset_id][path]; + } else { + return; + } + }, +}; + +const actions = { + fetchPathDestination: async ({ commit }, { history_dataset_id, path }) => { + const data = await getPathDestination(history_dataset_id, path); + commit("savePathDestination", { history_dataset_id, path, pathDestination: data }); + }, +}; + +const mutations = { + savePathDestination: (state, { history_dataset_id, path, pathDestination }) => { + state.datasetPathDestination[history_dataset_id] = { [path]: pathDestination }; + }, +}; + +export const datasetPathDestinationStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/galaxy/scripts/store/index.js b/client/galaxy/scripts/store/index.js index 0f89419a996..8b05dc88eb0 100644 --- a/client/galaxy/scripts/store/index.js +++ b/client/galaxy/scripts/store/index.js @@ -15,6 +15,7 @@ import { historyStore } from "./historyStore"; import { userStore } from "./userStore"; import { configStore } from "./configStore"; import { workflowStore } from "./workflowStore"; +import { datasetPathDestinationStore } from "./datasetPathDestinationStore"; Vue.use(Vuex); @@ -33,6 +34,7 @@ export function createStore() { tags: tagStore, jobMetrics: jobMetricsStore, destinationParameters: jobDestinationParametersStore, + datasetPathDestination: datasetPathDestinationStore, invocations: invocationStore, user: userStore, config: configStore, From 836b0063cabde24829908e0686c22a2eb967b98e Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Wed, 27 May 2020 20:14:21 +0200 Subject: [PATCH 044/188] store for ext files --- .../Dataset/DatasetLink/DatasetLink.vue | 2 +- .../Dataset/compositeDatasetUtils.js | 21 ++++++++--- .../scripts/store/datasetExtFilesStore.js | 35 +++++++++++++++++++ client/galaxy/scripts/store/index.js | 2 ++ 4 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 client/galaxy/scripts/store/datasetExtFilesStore.js diff --git a/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue b/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue index 15be735a64a..c3a4411734d 100644 --- a/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue +++ b/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue @@ -27,7 +27,7 @@ export default { this.pathDestination = {}; if (this.path && this.path !== "undefined") { // download individual file from composite dataset - this.fetchPathDestination({ history_dataset_id: this.history_dataset_id, path: this.path }).then((resp) => { + this.fetchPathDestination({ history_dataset_id: this.history_dataset_id, path: this.path }).then(() => { this.pathDestination = this.$store.getters.pathDestination(this.history_dataset_id, this.path); }); } else { diff --git a/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js b/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js index d8580a666e5..67a0bdf5f27 100644 --- a/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js +++ b/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js @@ -1,11 +1,13 @@ import { getAppRoot } from "onload/loadConfig"; import { Services } from "components/Dataset/services"; +import store from "../../store/index"; -export const getPathDestination = (history_dataset_id, path) => { +export const getPathDestination = async (history_dataset_id, path) => { const services = new Services({ root: getAppRoot() }); - return services.getCompositeDatasetContentFiles(history_dataset_id).then((datasetContent) => { - const pathDestination = { datasetContent: datasetContent }; - if (datasetContent[0].class === "Directory") pathDestination.datasetRootDir = datasetContent[0].path; + + const computePathDestination = (pathDestination) => { + if (pathDestination.datasetContent[0].class === "Directory") + pathDestination.datasetRootDir = datasetContent[0].path; else return; if (path === undefined || path === "undefined") { @@ -27,5 +29,14 @@ export const getPathDestination = (history_dataset_id, path) => { pathDestination.fileLink = services.getCompositeDatasetLink(history_dataset_id, datasetEntry.path); } return pathDestination; - }); + }; + + let datasetContent = store.getters.getDatasetExtFiles(history_dataset_id); + + if (datasetContent == null) { + await store.dispatch("fetchDatasetExtFiles", history_dataset_id); + datasetContent = store.getters.getDatasetExtFiles(history_dataset_id); + } + + return computePathDestination({ datasetContent: datasetContent }); }; diff --git a/client/galaxy/scripts/store/datasetExtFilesStore.js b/client/galaxy/scripts/store/datasetExtFilesStore.js new file mode 100644 index 00000000000..c07cecb1d85 --- /dev/null +++ b/client/galaxy/scripts/store/datasetExtFilesStore.js @@ -0,0 +1,35 @@ +export const state = { + datasetExtFilesById: {}, +}; + +import Vue from "vue"; +import { getAppRoot } from "onload/loadConfig"; +import axios from "axios"; + +const getters = { + getDatasetExtFiles: (state) => (history_dataset_id) => { + return state.datasetExtFilesById[history_dataset_id] || null; + }, +}; + +const actions = { + fetchDatasetExtFiles: async ({ commit }, history_dataset_id) => { + const { data } = await axios.get( + `${getAppRoot()}api/histories/${history_dataset_id}/contents/${history_dataset_id}/extra_files` + ); + commit("saveDatasetExtFiles", { history_dataset_id, datasetExtFiles: data }); + }, +}; + +const mutations = { + saveDatasetExtFiles: (state, { history_dataset_id, datasetExtFiles }) => { + Vue.set(state.datasetExtFilesById, history_dataset_id, datasetExtFiles); + }, +}; + +export const datasetExtFilesStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/galaxy/scripts/store/index.js b/client/galaxy/scripts/store/index.js index 8b05dc88eb0..9554dd86af8 100644 --- a/client/galaxy/scripts/store/index.js +++ b/client/galaxy/scripts/store/index.js @@ -16,6 +16,7 @@ import { userStore } from "./userStore"; import { configStore } from "./configStore"; import { workflowStore } from "./workflowStore"; import { datasetPathDestinationStore } from "./datasetPathDestinationStore"; +import { datasetExtFilesStore } from "./datasetExtFilesStore"; Vue.use(Vuex); @@ -35,6 +36,7 @@ export function createStore() { jobMetrics: jobMetricsStore, destinationParameters: jobDestinationParametersStore, datasetPathDestination: datasetPathDestinationStore, + datasetExtFiles: datasetExtFilesStore, invocations: invocationStore, user: userStore, config: configStore, From 24e3f639ebf7476ef5679de8755cb786442f1743 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Wed, 27 May 2020 20:21:04 +0200 Subject: [PATCH 045/188] adopt dataset_index and dataset_as_index --- .../components/Dataset/DatasetAsImage/DatasetAsImage.vue | 8 ++++++-- .../components/Dataset/DatasetIndex/DatasetIndex.vue | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue b/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue index 955ee315bd6..3847b70fa31 100644 --- a/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue +++ b/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue @@ -9,7 +9,7 @@ diff --git a/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue b/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue index efe37be8130..762e014b574 100644 --- a/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue +++ b/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue @@ -16,7 +16,7 @@ diff --git a/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue b/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue index 46aa38e7a8c..0acf584097c 100644 --- a/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue +++ b/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue @@ -1,8 +1,5 @@ @@ -16,14 +16,38 @@ // Not really a very generic ProgressBar - consider renaming to StateProgressBar. export default { props: { - total: { type: Number, default: 1 }, - note: { type: String, default: null }, - loading: { type: Boolean, default: false }, - infoCount: { type: Number, default: 0 }, - okCount: { type: Number, default: 0 }, - runningCount: { type: Number, default: 0 }, - newCount: { type: Number, default: 0 }, - errorCount: { type: Number, default: 0 }, - } + total: { + type: Number, + default: 1, + }, + note: { + type: String, + default: null, + }, + loading: { + type: Boolean, + default: false, + }, + infoCount: { + type: Number, + default: 0, + }, + okCount: { + type: Number, + default: 0, + }, + runningCount: { + type: Number, + default: 0, + }, + newCount: { + type: Number, + default: 0, + }, + errorCount: { + type: Number, + default: 0, + }, + }, }; diff --git a/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue b/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue index 0acf584097c..759556c4992 100644 --- a/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue +++ b/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue @@ -1,60 +1,57 @@ - From 331cc4e4b3aedf1c6a0f106bb60acdd0f0494c57 Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 9 Jul 2020 12:55:26 -0400 Subject: [PATCH 153/188] Remove redundant template wrappers, add spinner to invocation view --- .../WorkflowInvocationState.vue | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue b/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue index 759556c4992..c5874f6e172 100644 --- a/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue +++ b/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue @@ -1,7 +1,7 @@