From d9f68d74a4cf7c6b87defe94f6724342d1ebdb50 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 5 Nov 2022 17:04:18 +0100 Subject: [PATCH 001/258] Create traceback.txt when external metadata fails It's always a little tricky to debug this, and often you'll have to run the metadata script by hand activating galaxy's virtualenv and setting PYTHONPATH to galaxy's lib folder. This should improve the situation a lot, both in production and when debugging tests. Should help (a little) with https://github.com/galaxyproject/galaxy/issues/14931 (but this is in celery, so we wouldn't pass through this code). --- lib/galaxy/metadata/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/metadata/__init__.py b/lib/galaxy/metadata/__init__.py index f5d46ed0296..22ef4868ba8 100644 --- a/lib/galaxy/metadata/__init__.py +++ b/lib/galaxy/metadata/__init__.py @@ -17,7 +17,24 @@ from galaxy.util import safe_makedirs log = getLogger(__name__) -SET_METADATA_SCRIPT = "from galaxy_ext.metadata.set_metadata import set_metadata; set_metadata()" +SET_METADATA_SCRIPT = """ +import os +import traceback +try: + from galaxy_ext.metadata.set_metadata import set_metadata; set_metadata() +except Exception: + WORKING_DIRECTORY = os.getcwd() + WORKING_PARENT = os.path.join(WORKING_DIRECTORY, os.path.pardir) + if not os.path.isdir("working") and os.path.isdir(os.path.join(WORKING_PARENT, "working")): + # We're probably in pulsar + WORKING_DIRECTORY = WORKING_PARENT + METADATA_DIRECTORY = os.path.join(WORKING_DIRECTORY, "metadata") + EXPORT_STORE_DIRECTORY = os.path.join(METADATA_DIRECTORY, "outputs_populated") + os.makedirs(EXPORT_STORE_DIRECTORY, exist_ok=True) + with open(os.path.join(EXPORT_STORE_DIRECTORY, "traceback.txt"), "w") as out: + out.write(traceback.format_exc()) + raise +""" def get_metadata_compute_strategy(config, job_id, metadata_strategy_override=None, tool_id=None): From 67608c274e5f18128fe8a112ae21685987eef172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9r=C3=A9nice=20Batut?= Date: Tue, 8 Nov 2022 17:11:57 +0100 Subject: [PATCH 002/258] Add interactive tool for Mgnify Jupyter lab --- .../interactivetool_mgnify_notebook.xml | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 tools/interactive/interactivetool_mgnify_notebook.xml diff --git a/tools/interactive/interactivetool_mgnify_notebook.xml b/tools/interactive/interactivetool_mgnify_notebook.xml new file mode 100644 index 00000000000..883ab9f17d8 --- /dev/null +++ b/tools/interactive/interactivetool_mgnify_notebook.xml @@ -0,0 +1,122 @@ + + + quay.io/microbiome-informatics/emg-notebooks.dev:latest + + + + 8888 + lab + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `_'s datasets using Python or with R using the `MGnifyR `_ package. + +Why such notebooks? +------------------- + +The quantity and richness of `metagenomics-derived data `_ in MGnify grows every day. The `MGnify website `_ is the best place to start exploring and searching the MGnify database, and allows users to download modest query results as CSV tables. + +For larger queries, or more complex requirements like fetching metadata from samples across multiple studies, a programmatic access approach is far better. + +Programmatic access - fetching data from MGnify using a terminal command or code script - uses the `MGnify API `_ (`Application Programming Interface `_). The API provides access to every data type in MGnify: `Studies `_, `Samples `_, `Analyses `_, `Annotations `_, `MAGs `_ etc: it is what lies behind the MGnify website. Using the API means you can fetch more data than is possible via the website, and can help you write reproducible analysis scripts. + +The API can be explored interactively online, using the `API Browser `_. But actually using the API first requires knowledge and/or installation of tools on your computer. This might range from a command line tool like `cURL `_, to learning R and setting up the `R Studio `_ application, to setting up a `Python `_ environment and installing a suite of `packages used for data analysis `_. Second, the API returns most data in `JSON format `_: this is standard on the web, but less familiar for bioinformaticians used to TSVs and dataframes. + +The `MGnify Notebook Server `_ and `MGnifyR package `_ are designed to bridge these gaps. Users can launch an online R and Python coding environment in their browser, without installing anything. The environment is hosted by EMBL's `Cell Biology and Biophysics Computational Support team `_, who support computational projects across EMBL. It already includes the main libraries needed for communicating with the MGnify API, analysing data, and making plots. It uses the popular `Jupyter Lab `_ software, which means you can code inside `Notebooks `_: interactive code documents. + +There are example Notebooks written in both R and Python, so users can pick whichever they're more familiar with. +]]> + From d6a2feb270dd5017a5fba5343e0668a0d789ab31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 9 Nov 2022 15:00:42 +0100 Subject: [PATCH 003/258] remove not used inputs --- .../interactivetool_mgnify_notebook.xml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tools/interactive/interactivetool_mgnify_notebook.xml b/tools/interactive/interactivetool_mgnify_notebook.xml index 883ab9f17d8..62d5dc1e408 100644 --- a/tools/interactive/interactivetool_mgnify_notebook.xml +++ b/tools/interactive/interactivetool_mgnify_notebook.xml @@ -13,8 +13,6 @@ $__galaxy_url__ 8080 $__galaxy_url__ - true - - @@ -102,6 +94,7 @@ `_'s datasets using Python or with R using the `MGnifyR `_ package. Why such notebooks? @@ -118,5 +111,5 @@ The API can be explored interactively online, using the `API Browser `_ and `MGnifyR package `_ are designed to bridge these gaps. Users can launch an online R and Python coding environment in their browser, without installing anything. The environment is hosted by EMBL's `Cell Biology and Biophysics Computational Support team `_, who support computational projects across EMBL. It already includes the main libraries needed for communicating with the MGnify API, analysing data, and making plots. It uses the popular `Jupyter Lab `_ software, which means you can code inside `Notebooks `_: interactive code documents. There are example Notebooks written in both R and Python, so users can pick whichever they're more familiar with. -]]> + ]]> From 0f9a364396897b5f1ef3b937770f7609655d0122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 15 Nov 2022 23:06:20 +0100 Subject: [PATCH 004/258] used tag container version --- tools/interactive/interactivetool_mgnify_notebook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/interactive/interactivetool_mgnify_notebook.xml b/tools/interactive/interactivetool_mgnify_notebook.xml index 62d5dc1e408..12186ec9ce3 100644 --- a/tools/interactive/interactivetool_mgnify_notebook.xml +++ b/tools/interactive/interactivetool_mgnify_notebook.xml @@ -1,6 +1,6 @@ - quay.io/microbiome-informatics/emg-notebooks.dev:latest + quay.io/microbiome-informatics/emg-notebooks.dev:v1.0.0 From 09744c79a55efeb6119e7d1020928377e798df84 Mon Sep 17 00:00:00 2001 From: hujambo-dunia Date: Mon, 21 Nov 2022 11:02:29 -0500 Subject: [PATCH 005/258] Added static Email Address in order (1) to prevent Form Field Manipulation Hack and (2) to prevent non-owned/non-verified Email Addresses. --- .../DatasetInformation/DatasetError.test.js | 1 - .../DatasetInformation/DatasetError.vue | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/client/src/components/DatasetInformation/DatasetError.test.js b/client/src/components/DatasetInformation/DatasetError.test.js index 39c8b513a29..8d2c7d78f55 100644 --- a/client/src/components/DatasetInformation/DatasetError.test.js +++ b/client/src/components/DatasetInformation/DatasetError.test.js @@ -48,7 +48,6 @@ describe("DatasetError", () => { expect(messages.at(1).text()).toBe("message_2"); expect(wrapper.find("#dataset-error-has-empty-inputs")).toBeDefined(); expect(wrapper.find("#dataset-error-has-duplicate-inputs")).toBeDefined(); - expect(wrapper.findAll("#dataset-error-email").length).toBe(1); }); it("check props without common problems", async () => { diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue index 29b1888a052..69d33eb4768 100644 --- a/client/src/components/DatasetInformation/DatasetError.vue +++ b/client/src/components/DatasetInformation/DatasetError.vue @@ -62,11 +62,8 @@ >{{ resultMessage[0] }}
- + {{ emailTitle }} + {{ currentUserEmail }}