From bfffad08620cc9f8aa12224ec41d88e1a6a3e4ad Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 15 Oct 2021 19:31:44 +0200 Subject: [PATCH 001/781] Tool linter: simple test for test params absent from inputs --- lib/galaxy/tool_util/linters/tests.py | 24 ++++++++++++ test/unit/tool_util/test_tool_linters.py | 50 +++++++++++++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/linters/tests.py b/lib/galaxy/tool_util/linters/tests.py index 562ae880aee..f5ae7dae004 100644 --- a/lib/galaxy/tool_util/linters/tests.py +++ b/lib/galaxy/tool_util/linters/tests.py @@ -26,6 +26,30 @@ def lint_tsts(tool_xml, lint_ctx): has_test = True break + # really simple test that test parameters are also present in the inputs + for param in test.findall("param"): + name = param.attrib.get("name", None) + if not name: + lint_ctx.warn("Found test param tag without a name defined.") + continue + name = name.split("|")[-1] + xpaths = [f"@name='{name}'", + f"@argument='{name}'", + f"@argument='-{name}'", + f"@argument='--{name}'"] + if "_" in name: + xpaths += [f"@argument='-{name.replace('_', '-')}'", + f"@argument='--{name.replace('_', '-')}'"] + found = False + for xp in xpaths: + inxpath = f".//inputs//param[{xp}]" + inparam = tool_xml.findall(inxpath) + if len(inparam) > 0: + found = True + break + if not found: + lint_ctx.error(f"Test param {name} not found in the inputs") + output_data_names, output_collection_names = _collect_output_names(tool_xml) found_output_test = False for output in test.findall("output"): diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py index 88dbbc19e03..49809b809ff 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters.py @@ -4,7 +4,8 @@ from galaxy.tool_util.lint import LintContext from galaxy.tool_util.linters import ( general, inputs, - outputs + outputs, + tests ) from galaxy.tool_util.parser.xml import XmlToolSource from galaxy.util import etree @@ -174,6 +175,36 @@ OUTPUTS_DISCOVER_TOOL_PROVIDED_METADATA = """ """ +# check that linter does complain about tests wo assumptions +TESTS_WO_EXPECTATIONS = """ + + + + + + +""" + +TESTS_PARAM = """ + + + + + + + + + + + + + + + + + +""" + TESTS = [ ( WHITESPACE_IN_VERSIONS_AND_NAMES, general.lint_general, @@ -260,6 +291,19 @@ TESTS = [ lambda x: len(x.warn_messages) == 0 and len(x.error_messages) == 0 ), + ( + TESTS_WO_EXPECTATIONS, tests.lint_tsts, + lambda x: + 'No outputs or expectations defined for tests, this test is likely invalid.' in x.warn_messages + and 'No valid test(s) found.' in x.warn_messages + and len(x.warn_messages) == 2 and len(x.error_messages) == 0 + ), + ( + TESTS_PARAM, tests.lint_tsts, + lambda x: + "Test param non_existent_test_name not found in the inputs" in x.error_messages + and len(x.warn_messages) == 0 and len(x.error_messages) == 1 + ) ] TEST_IDS = [ @@ -273,7 +317,9 @@ TEST_IDS = [ 'select option definitions', 'validator imcompatibilities', 'outputs collection static elements with format_source', - 'outputs discover datatsets with tool provided metadata' + 'outputs discover datatsets with tool provided metadata', + 'test without expectations', + 'test param missing from inputs' ] From 101d956b2cd353e46def09e7f632faa5055a7de2 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Tue, 2 Nov 2021 19:22:33 +0530 Subject: [PATCH 002/781] Use correct repo name on quay.io --- .github/workflows/build_container_image.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_container_image.yaml b/.github/workflows/build_container_image.yaml index e4d5f1f6922..0b9d4aa42ca 100644 --- a/.github/workflows/build_container_image.yaml +++ b/.github/workflows/build_container_image.yaml @@ -22,7 +22,7 @@ jobs: id: branch run: echo "::set-output name=name::$(BRANCH_NAME=${GITHUB_REF##*/}; echo ${BRANCH_NAME/release_/}-auto)" - run: docker build . -t galaxy/galaxy-min:${{ steps.vars.outputs.sha_short }} -f .k8s_ci.Dockerfile - - run: docker tag galaxy/galaxy-min:${{ steps.vars.outputs.sha_short }} quay.io/galaxy-k8s/galaxy:${{ steps.branch.outputs.name }} && docker tag galaxy/galaxy-min:${{ steps.vars.outputs.sha_short }} quay.io/galaxy-k8s/galaxy:${{ steps.vars.outputs.sha_short }} + - run: docker tag galaxy/galaxy-min:${{ steps.vars.outputs.sha_short }} quay.io/galaxy-min/galaxy:${{ steps.branch.outputs.name }} && docker tag galaxy/galaxy-min:${{ steps.vars.outputs.sha_short }} quay.io/galaxy-min/galaxy:${{ steps.vars.outputs.sha_short }} - name: Login to quay.io uses: actions-hub/docker/login@master env: @@ -32,8 +32,8 @@ jobs: - name: Push to quay.io with commit ID uses: actions-hub/docker@master with: - args: push quay.io/galaxy-k8s/galaxy:${{ steps.vars.outputs.sha_short }} + args: push quay.io/galaxy-min/galaxy:${{ steps.vars.outputs.sha_short }} - name: Push to quay.io with branch name uses: actions-hub/docker@master with: - args: push quay.io/galaxy-k8s/galaxy:${{ steps.branch.outputs.name }} + args: push quay.io/galaxy-min/galaxy:${{ steps.branch.outputs.name }} From 33af0e39520c125367ef8171739010caaf49f0cb Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Tue, 2 Nov 2021 19:24:09 +0530 Subject: [PATCH 003/781] Only tag by branch name for now, to avoid too many images --- .dockerignore | 1 + .github/workflows/build_container_image.yaml | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index d4e31f2c7a2..ce3827197e7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ .k8s_ci.Dockerfile .venv database +node_modules diff --git a/.github/workflows/build_container_image.yaml b/.github/workflows/build_container_image.yaml index 0b9d4aa42ca..e88454327ce 100644 --- a/.github/workflows/build_container_image.yaml +++ b/.github/workflows/build_container_image.yaml @@ -21,18 +21,14 @@ jobs: - name: Set branch name id: branch run: echo "::set-output name=name::$(BRANCH_NAME=${GITHUB_REF##*/}; echo ${BRANCH_NAME/release_/}-auto)" - - run: docker build . -t galaxy/galaxy-min:${{ steps.vars.outputs.sha_short }} -f .k8s_ci.Dockerfile - - run: docker tag galaxy/galaxy-min:${{ steps.vars.outputs.sha_short }} quay.io/galaxy-min/galaxy:${{ steps.branch.outputs.name }} && docker tag galaxy/galaxy-min:${{ steps.vars.outputs.sha_short }} quay.io/galaxy-min/galaxy:${{ steps.vars.outputs.sha_short }} + - run: docker build . -t galaxy/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile + - run: docker tag galaxy/galaxy-min:${{ steps.branch.outputs.name }} quay.io/galaxy-min/galaxy:${{ steps.branch.outputs.name }} - name: Login to quay.io uses: actions-hub/docker/login@master env: DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} DOCKER_PASSWORD: ${{ secrets.QUAY_PASSWORD }} DOCKER_REGISTRY_URL: quay.io - - name: Push to quay.io with commit ID - uses: actions-hub/docker@master - with: - args: push quay.io/galaxy-min/galaxy:${{ steps.vars.outputs.sha_short }} - name: Push to quay.io with branch name uses: actions-hub/docker@master with: From 9d54c2b134584727f3d3cc1c174d6b8597685915 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Tue, 2 Nov 2021 20:53:01 +0530 Subject: [PATCH 004/781] Remove -auto from end of tag --- .github/workflows/build_container_image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_container_image.yaml b/.github/workflows/build_container_image.yaml index e88454327ce..0dd9c572d0e 100644 --- a/.github/workflows/build_container_image.yaml +++ b/.github/workflows/build_container_image.yaml @@ -20,7 +20,7 @@ jobs: run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" - name: Set branch name id: branch - run: echo "::set-output name=name::$(BRANCH_NAME=${GITHUB_REF##*/}; echo ${BRANCH_NAME/release_/}-auto)" + run: echo "::set-output name=name::$(BRANCH_NAME=${GITHUB_REF##*/}; echo ${BRANCH_NAME/release_/})" - run: docker build . -t galaxy/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile - run: docker tag galaxy/galaxy-min:${{ steps.branch.outputs.name }} quay.io/galaxy-min/galaxy:${{ steps.branch.outputs.name }} - name: Login to quay.io From e5b50d147cdbc092f3f1c9de3b09a717b646b627 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Tue, 2 Nov 2021 21:19:16 +0530 Subject: [PATCH 005/781] Add open container labels to image --- .github/workflows/build_container_image.yaml | 2 +- .k8s_ci.Dockerfile | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_container_image.yaml b/.github/workflows/build_container_image.yaml index 0dd9c572d0e..830ff86d435 100644 --- a/.github/workflows/build_container_image.yaml +++ b/.github/workflows/build_container_image.yaml @@ -21,7 +21,7 @@ jobs: - name: Set branch name id: branch run: echo "::set-output name=name::$(BRANCH_NAME=${GITHUB_REF##*/}; echo ${BRANCH_NAME/release_/})" - - run: docker build . -t galaxy/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile + - run: docker build . --build-arg GIT_COMMIT=$(git rev-parse HEAD) --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg IMAGE_TAG=${{ steps.branch.outputs.name }} --build-arg IMAGE_TAG=${{ steps.branch.outputs.name }} -t galaxy/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile - run: docker tag galaxy/galaxy-min:${{ steps.branch.outputs.name }} quay.io/galaxy-min/galaxy:${{ steps.branch.outputs.name }} - name: Login to quay.io uses: actions-hub/docker/login@master diff --git a/.k8s_ci.Dockerfile b/.k8s_ci.Dockerfile index 931ac1a2a68..d7d2c90a213 100644 --- a/.k8s_ci.Dockerfile +++ b/.k8s_ci.Dockerfile @@ -27,6 +27,10 @@ ARG FINAL_STAGE_BASE=$STAGE1_BASE ARG GALAXY_USER=galaxy ARG GALAXY_PLAYBOOK_REPO=https://github.com/galaxyproject/galaxy-docker-k8s +ARG GIT_COMMIT=unspecified +ARG BUILD_DATE=unspecified +ARG IMAGE_TAG=unspecified + #====================================================== # Stage 1 - Setup common requirements for build #====================================================== @@ -124,6 +128,22 @@ ARG ROOT_DIR ARG SERVER_DIR ARG GALAXY_USER +ARG GIT_COMMIT +ARG BUILD_DATE +ARG IMAGE_TAG + +LABEL org.opencontainers.image.title="Galaxy Minimal Image" \ + org.opencontainers.image.description="A size optimized image for Galaxy targeting k8s and ci applications" \ + org.opencontainers.image.authors="galaxyproject.org" \ + org.opencontainers.image.vendor="Galaxy Project" \ + org.opencontainers.image.documentation="https://github.com/galaxyproject/galaxy-docker-k8s" \ + org.opencontainers.image.licenses="MIT" \ + org.opencontainers.image.version="$IMAGE_TAG" \ + org.opencontainers.image.url="https://github.com/galaxyproject/galaxy-docker-k8s" \ + org.opencontainers.image.source="https://github.com/galaxyproject/galaxy.git" \ + org.opencontainers.image.revision=$GIT_COMMIT \ + org.opencontainers.image.created=$BUILD_DATE + # Init Env ENV LC_ALL=en_US.UTF-8 ENV LANG=en_US.UTF-8 From 6a54fe097334db3176dc335d4920374c56007ac9 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Wed, 3 Nov 2021 22:01:14 -0400 Subject: [PATCH 006/781] Change anvil branche to dev --- .github/workflows/build_container_image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_container_image.yaml b/.github/workflows/build_container_image.yaml index 830ff86d435..3244f9e82d9 100644 --- a/.github/workflows/build_container_image.yaml +++ b/.github/workflows/build_container_image.yaml @@ -3,7 +3,7 @@ on: push: branches: - 'release*' - - anvil + - dev concurrency: group: docker-build-${{ github.ref }} cancel-in-progress: true From 48d54317c053a7123b522abcd97732b93e4e747f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 4 Nov 2021 09:32:10 +0100 Subject: [PATCH 007/781] Fix action token sending for chracters that are escaped in html The function body of escape is ``` def escape(s): """Replace the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in the string with HTML-safe sequences. Use this if you need to display text that might contain such characters in HTML. If the object has an ``__html__`` method, it is called and the return value is assumed to already be safe for HTML. :param s: An object to be converted to a string and escaped. :return: A :class:`Markup` string with the escaped text. """ if hasattr(s, "__html__"): return Markup(s.__html__()) return Markup( text_type(s) .replace("&", "&") .replace(">", ">") .replace("<", "<") .replace("'", "'") .replace('"', """) ) ``` It doesn't make sense to lookup the replaced values in the database, and it breaks single-quotes in the email username portion, which is valid https://en.wikipedia.org/wiki/Email_address#Syntax --- lib/galaxy/managers/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index cb9f4c67b73..0cef6a3bea0 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -488,7 +488,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): """ Send the verification email containing the activation link to the user's email. """ - activation_token = self.__get_activation_token(trans, escape(email)) + activation_token = self.__get_activation_token(trans, email) activation_link = url_for(controller='user', action='activate', activation_token=activation_token, email=escape(email), qualified=True) host = self.__get_host(trans) custom_message = '' From a1d475e0d043eed4f1cc223982f899ea459cb053 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 5 Nov 2021 15:18:37 +0100 Subject: [PATCH 008/781] Reduce duplication of history_id and _event_start handling in upload modal variants --- client/src/components/Upload/Collection.vue | 30 --------------- client/src/components/Upload/Default.vue | 32 ---------------- .../src/components/Upload/UploadBoxMixin.js | 38 +++++++++++++++++-- 3 files changed, 34 insertions(+), 66 deletions(-) diff --git a/client/src/components/Upload/Collection.vue b/client/src/components/Upload/Collection.vue index 89f97a755a5..0c32cac201a 100644 --- a/client/src/components/Upload/Collection.vue +++ b/client/src/components/Upload/Collection.vue @@ -216,14 +216,6 @@ export default { appModel() { return this.app.model; }, - history_id() { - const storeId = this.$store?.getters["betaHistory/currentHistoryId"]; - if (storeId) { - return storeId; - } - const legacyId = this.app.currentHistory(); - return legacyId; - }, }, watch: { extension: function (value) { @@ -271,28 +263,6 @@ export default { this.$emit("dismiss"); }, - /** Start upload process */ - _eventStart: function () { - if (this.counterAnnounce == 0 || this.counterRunning > 0) { - return; - } - this.uploadSize = 0; - this.uploadCompleted = 0; - this.collection.each((model) => { - if (model.get("status") == "init") { - model.set("status", "queued"); - this.uploadSize += model.get("file_size"); - } - }); - this.appModel.set({ percentage: 0, status: "success" }); - this.counterRunning = this.counterAnnounce; - - // package ftp files separately, and remove them from queue - this._uploadFtp(); - this.uploadbox.start(); - this._updateStateForCounters(); - }, - /** Remove all */ _eventReset: function () { if (this.counterRunning == 0) { diff --git a/client/src/components/Upload/Default.vue b/client/src/components/Upload/Default.vue index 3f2c78d146d..9b24860b252 100644 --- a/client/src/components/Upload/Default.vue +++ b/client/src/components/Upload/Default.vue @@ -213,13 +213,6 @@ export default { appModel() { return this.app.model; }, - history_id() { - const storeId = this.$store?.getters["betaHistory/currentHistoryId"]; - if (storeId) { - return storeId; - } - return this.app.currentHistoryId; - }, }, watch: { extension: function (value) { @@ -262,31 +255,6 @@ export default { this._updateStateForSuccess(it); }, - /** Start upload process */ - _eventStart: function () { - if (this.counterAnnounce !== 0 && this.counterRunning === 0) { - // prepare upload process - this.uploadSize = 0; - this.uploadCompleted = 0; - this.collection.each((model) => { - if (model.get("status") == "init") { - model.set("status", "queued"); - this.uploadSize += model.get("file_size"); - } - }); - - this.appModel.set({ percentage: 0, status: "success" }); - this.counterRunning = this.counterAnnounce; - - // package ftp files separately, and remove them from queue - this._uploadFtp(); - - // queue remaining files - this.uploadbox.start(); - this._updateStateForCounters(); - } - }, - /** Remove all */ _eventReset: function () { if (this.counterRunning === 0) { diff --git a/client/src/components/Upload/UploadBoxMixin.js b/client/src/components/Upload/UploadBoxMixin.js index 807ea1d189d..d300bc0c029 100644 --- a/client/src/components/Upload/UploadBoxMixin.js +++ b/client/src/components/Upload/UploadBoxMixin.js @@ -14,6 +14,7 @@ import { getAppRoot } from "onload"; import axios from "axios"; const localize = _l; +export const defaultNewFileName = "New File"; export default { components: { @@ -53,6 +54,14 @@ export default { btnCloseTitle() { return this.hasCallback ? "Cancel" : "Close"; }, + history_id() { + const storeId = this.$store?.getters["betaHistory/currentHistoryId"]; + if (storeId) { + return storeId; + } + const legacyId = this.app.currentHistory(); + return legacyId; + }, }, methods: { $uploadBox() { @@ -76,6 +85,29 @@ export default { uploadSelect: function () { this.uploadbox.select(); }, + + /** Start upload process */ + _eventStart: function () { + if (this.counterAnnounce == 0 || this.counterRunning > 0) { + return; + } + this.uploadSize = 0; + this.uploadCompleted = 0; + this.collection.each((model) => { + if (model.get("status") == "init") { + model.set("status", "queued"); + this.uploadSize += model.get("file_size"); + } + }); + this.appModel.set({ percentage: 0, status: "success" }); + this.counterRunning = this.counterAnnounce; + + // package ftp files separately, and remove them from queue + this._uploadFtp(); + this.uploadbox.start(); + this._updateStateForCounters(); + }, + /** Package and upload ftp files in a single request */ _uploadFtp: function () { const list = []; @@ -256,7 +288,7 @@ export default { /** Create a new file */ _eventCreate: function (withNewFile) { if (withNewFile == true) { - this.uploadbox.add([{ name: "New File", size: 0, mode: "new" }]); + this.uploadbox.add([{ name: defaultNewFileName, size: 0, mode: "new" }]); } else if (withNewFile == false) { this.uploadbox.add([{ size: 0, mode: "new" }]); } @@ -346,9 +378,7 @@ export default { return models; }, getRequestUrl: function (items, history_id) { - var data = this.app.toData(items, history_id); - const appRoot = getAppRoot(); - return data.fetchRequest ? `${appRoot}api/tools/fetch` : this.app.uploadPath; + return `${getAppRoot()}api/tools/fetch`; }, }, }; From d92bf5d7ae21a5bfcce0f7b36b93e4c1c37d4d63 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 5 Nov 2021 15:20:05 +0100 Subject: [PATCH 009/781] Drop legacy upload1 / uploadpost handling --- client/src/components/Upload/Collection.vue | 3 +- client/src/components/Upload/Composite.vue | 5 +- client/src/components/Upload/Default.vue | 3 +- .../src/components/Upload/UploadBoxMixin.js | 4 +- .../components/Upload/UploadModalContent.vue | 139 +----------- client/src/components/Upload/helpers.js | 79 +++++++ client/src/utils/data.js | 19 +- client/src/utils/uploadbox.js | 197 ++---------------- 8 files changed, 119 insertions(+), 330 deletions(-) diff --git a/client/src/components/Upload/Collection.vue b/client/src/components/Upload/Collection.vue index 0c32cac201a..a68f9a481db 100644 --- a/client/src/components/Upload/Collection.vue +++ b/client/src/components/Upload/Collection.vue @@ -121,6 +121,7 @@ import _ from "underscore"; import { getGalaxyInstance } from "app"; import UploadRow from "mvc/upload/collection/collection-row"; import UploadBoxMixin from "./UploadBoxMixin"; +import { uploadModelsToPayload } from "./helpers"; import { BButton } from "bootstrap-vue"; export default { @@ -178,7 +179,7 @@ export default { this._eventAnnounce(index, file); }, initialize: (index) => { - return this.app.toData([this.collection.get(index)], this.history_id); + return uploadModelsToPayload([this.collection.get(index)], this.history_id); }, progress: (index, percentage) => { this._eventProgress(index, percentage); diff --git a/client/src/components/Upload/Composite.vue b/client/src/components/Upload/Composite.vue index dbb6a539828..0ae35809941 100644 --- a/client/src/components/Upload/Composite.vue +++ b/client/src/components/Upload/Composite.vue @@ -61,6 +61,7 @@ import $ from "jquery"; import { getGalaxyInstance } from "app"; import UploadRow from "mvc/upload/composite/composite-row"; import UploadBoxMixin from "./UploadBoxMixin"; +import { uploadModelsToPayload } from "./helpers"; export default { mixins: [UploadBoxMixin], @@ -139,9 +140,9 @@ export default { extension: this.extension, }); }); - $.uploadpost({ + $.uploadchunk({ url: this.app.uploadPath, - data: this.app.toFileUploadData(this.collection.filter()), + data: uploadModelsToPayload(this.collection.filter(), this.history_id, true), success: (message) => { this._eventSuccess(message); }, diff --git a/client/src/components/Upload/Default.vue b/client/src/components/Upload/Default.vue index 9b24860b252..ea05075faac 100644 --- a/client/src/components/Upload/Default.vue +++ b/client/src/components/Upload/Default.vue @@ -114,6 +114,7 @@ import _l from "utils/localization"; import _ from "underscore"; import UploadRow from "mvc/upload/default/default-row"; import UploadBoxMixin from "./UploadBoxMixin"; +import { uploadModelsToPayload } from "./helpers"; import { BButton } from "bootstrap-vue"; export default { @@ -175,7 +176,7 @@ export default { this._eventAnnounce(index, file); }, initialize: (index) => { - return this.app.toData([this.collection.get(index)], this.history_id); + return uploadModelsToPayload([this.collection.get(index)], this.history_id); }, progress: (index, percentage) => { this._eventProgress(index, percentage); diff --git a/client/src/components/Upload/UploadBoxMixin.js b/client/src/components/Upload/UploadBoxMixin.js index d300bc0c029..94e751dbc2e 100644 --- a/client/src/components/Upload/UploadBoxMixin.js +++ b/client/src/components/Upload/UploadBoxMixin.js @@ -5,6 +5,7 @@ import Popover from "mvc/ui/ui-popover"; import UploadExtension from "mvc/upload/upload-extension"; import UploadModel from "mvc/upload/upload-model"; import UploadWrapper from "./UploadWrapper"; +import { defaultNewFileName, uploadModelsToPayload } from "./helpers"; import { getGalaxyInstance } from "app"; import UploadFtp from "mvc/upload/upload-ftp"; import LazyLimited from "mvc/lazy/lazy-limited"; @@ -14,7 +15,6 @@ import { getAppRoot } from "onload"; import axios from "axios"; const localize = _l; -export const defaultNewFileName = "New File"; export default { components: { @@ -118,7 +118,7 @@ export default { } }); if (list.length > 0) { - const data = this.app.toFetchData(list, this.history_id); + const data = uploadModelsToPayload(list, this.history_id); axios .post(`${getAppRoot()}api/tools/fetch`, data) .then((message) => { diff --git a/client/src/components/Upload/UploadModalContent.vue b/client/src/components/Upload/UploadModalContent.vue index adb7a993247..91306caf186 100644 --- a/client/src/components/Upload/UploadModalContent.vue +++ b/client/src/components/Upload/UploadModalContent.vue @@ -34,6 +34,7 @@ import Collection from "./Collection"; import Default from "./Default"; import RulesInput from "./RulesInput"; import LoadingSpan from "components/LoadingSpan"; +import { uploadModelsToPayload } from "./helpers"; import { BTabs, BTab } from "bootstrap-vue"; import { commonProps } from "./helpers"; @@ -60,7 +61,6 @@ export default { extensionsSet: false, datatypesMapper: null, datatypesMapperReady: true, - URI_PREFIXES: ["http", "https", "ftp", "file", "gxfiles", "gximport", "gxuserimport", "gxftp"], }; }, created() { @@ -174,143 +174,12 @@ export default { currentFtp: function () { return this.currentUserId && this.ftpUploadSite; }, - toData: function (items, history_id) { - const data = { - fetchRequest: null, - uploadRequest: null, - }; - if (items && items.length > 0) { - const split = this.preprocess(items); - if (split.urls.length > 0) { - data.fetchRequest = this.toFetchData(split.urls, history_id); - } else { - data.uploadRequest = this.toFileUploadData(split.files, history_id); - } - } - return data; - }, - preprocess: function (items) { - const data = { - urls: [], - files: [], - }; - for (var index in items) { - var it = items[index]; - if (it.get("file_mode") != "new" || !this.itemIsURL(it)) { - data.files.push(it); - } else { - data.urls.push(it); - } - } - return data; - }, - itemIsURL: function (item) { - return this.URI_PREFIXES.some((prefix) => item.get("url_paste").startsWith(prefix)); - }, /** - * Package API data from array of models + * Package API data from array of backbone models * @param{Array} items - Upload items/rows filtered from a collection */ - toFileUploadData: function (items, history_id) { - // create dictionary for data submission - var data = { - payload: { - tool_id: "upload1", - history_id: history_id || this.currentHistoryId, - inputs: {}, - }, - files: [], - error_message: null, - }; - // add upload tools input data - if (items && items.length > 0) { - var inputs = { - file_count: items.length, - dbkey: items[0].get("genome", "?"), - // sometimes extension set to "" in automated testing after first upload of - // a session. https://github.com/galaxyproject/galaxy/issues/5169 - file_type: items[0].get("extension") || "auto", - }; - for (var index in items) { - var it = items[index]; - it.set("status", "running"); - if (it.get("file_size") > 0) { - var prefix = `files_${index}|`; - inputs[`${prefix}type`] = "upload_dataset"; - if (it.get("file_name") != "New File") { - inputs[`${prefix}NAME`] = it.get("file_name"); - } - inputs[`${prefix}space_to_tab`] = (it.get("space_to_tab") && "Yes") || null; - inputs[`${prefix}to_posix_lines`] = (it.get("to_posix_lines") && "Yes") || null; - inputs[`${prefix}dbkey`] = it.get("genome", null); - inputs[`${prefix}file_type`] = it.get("extension", null); - let uri; - let how; - switch (it.get("file_mode")) { - case "new": - inputs[`${prefix}url_paste`] = it.get("url_paste"); - break; - case "ftp": - uri = it.get("file_path"); - how = "ftp_files"; - if (uri.indexOf("://") >= 0) { - how = "url_paste"; - } - inputs[`${prefix}${how}`] = uri; - break; - case "local": - data.files.push({ - name: `${prefix}file_data`, - file: it.get("file_data"), - }); - } - } else if (it.get("optional")) { - continue; - } else { - data.error_message = "Upload content incomplete."; - it.set("status", "error"); - it.set("info", data.error_message); - break; - } - } - data.payload.inputs = JSON.stringify(inputs); - } - return data; - }, - toFetchData: function (items, history_id) { - var data = { - history_id: history_id, - space_to_tab: items[0].get("space_to_tab"), - to_posix_lines: items[0].get("to_posix_lines"), - targets: [ - { - destination: { type: "hdas" }, - elements: [], - name: "", - }, - ], - auto_decompress: true, - }; - - items.forEach((item) => { - let urls; - if (item.get("file_mode") == "ftp") { - urls = [item.get("file_uri") || item.get("file_path")]; - } else { - urls = item.get("url_paste").split("\n"); - } - urls.forEach((url) => { - if (url != "") { - data.targets[0].elements.push({ - url: url.trim(), - src: "url", - dbkey: item.get("genome", "?"), - ext: item.get("extension", "auto"), - }); - } - }); - }); - return data; + toData: function (items, history_id, composite = false) { + return uploadModelsToPayload(items, history_id, composite); }, }, }; diff --git a/client/src/components/Upload/helpers.js b/client/src/components/Upload/helpers.js index 0aca5362700..099bf93b6c3 100644 --- a/client/src/components/Upload/helpers.js +++ b/client/src/components/Upload/helpers.js @@ -1,4 +1,83 @@ import UploadUtils from "mvc/upload/upload-utils"; +export const defaultNewFileName = "New File"; + +const URI_PREFIXES = ["http", "https", "ftp", "file", "gxfiles", "gximport", "gxuserimport", "gxftp"]; +function itemIsUrl(item) { + return URI_PREFIXES.some((prefix) => item.get("url_paste").startsWith(prefix)); +} + +export function uploadModelsToPayload(items, history_id, composite = false) { + const files = []; + const elements = items + .map((item) => { + if (item.get("optional")) { + return null; + } + let src; + let pasteContent = null; + let fileName = item.get("file_name"); + if (fileName === defaultNewFileName) { + fileName = null; + } + const url = (item.get("file_uri") || item.get("file_path") || item.get("url_paste")).trim(); + switch (item.get("file_mode")) { + case "new": + if (itemIsUrl(item)) { + src = "url"; + } else { + pasteContent = item.get("url_paste"); + src = "pasted"; + } + break; + case "ftp": + if (url.indexOf("://") >= 0) { + src = "url"; + } + break; + case "local": + src = "files"; + files.push(item.get("file_data")); + break; + default: + console.error("Unknown file_mode", item); + } + return { + src: src, + url: url, + paste_content: pasteContent, + name: fileName, + dbkey: item.get("genome", "?"), + ext: item.get("extension", "auto"), + space_to_tab: item.get("space_to_tab"), + to_posix_lines: item.get("to_posix_lines"), + }; + }) + .filter((item) => item); + + const target = { + destination: { type: "hdas" }, + elements: elements, + }; + if (composite) { + const compositeItems = [ + { + src: "composite", + ext: elements[0].ext, + composite: { + items: target.elements, + }, + }, + ]; + delete target["elements"]; + target["items"] = compositeItems; + } + return { + history_id: history_id, + targets: [target], + auto_decompress: true, + files: files, + }; +} export const commonProps = { uploadPath: { diff --git a/client/src/utils/data.js b/client/src/utils/data.js index aa3642ad76c..1b0bb622ab7 100644 --- a/client/src/utils/data.js +++ b/client/src/utils/data.js @@ -5,6 +5,7 @@ import DataDialog from "components/DataDialog/DataDialog.vue"; import { FilesDialog } from "components/FilesDialog"; import DatasetCollectionDialog from "components/SelectionDialog/DatasetCollectionDialog.vue"; import { mountUploadModal } from "components/Upload"; +import { uploadModelsToPayload } from "components/Upload/helpers"; import { getGalaxyInstance } from "app"; import { getAppRoot } from "onload/loadConfig"; @@ -92,8 +93,8 @@ export function create(options) { return options.history_id; } getHistory().then((history_id) => { - $.uploadpost({ - url: `${getAppRoot()}api/tools`, + $.uploadchunk({ + url: `${getAppRoot()}api/tools/fetch`, success: (response) => { if (history_panel) { history_panel.refreshContents(); @@ -104,19 +105,7 @@ export function create(options) { }, error: options.error, data: { - payload: { - tool_id: "upload1", - history_id: history_id, - inputs: JSON.stringify({ - "files_0|type": "upload_dataset", - "files_0|NAME": options.file_name, - "files_0|space_to_tab": options.space_to_tab ? "Yes" : null, - "files_0|to_posix_lines": options.to_posix_lines ? "Yes" : null, - "files_0|dbkey": options.genome || "?", - "files_0|file_type": options.extension || "auto", - "files_0|url_paste": options.url_paste, - }), - }, + payload: uploadModelsToPayload([options], history_id), }, }); }); diff --git a/client/src/utils/uploadbox.js b/client/src/utils/uploadbox.js index 28b16f64cc2..a0671bd1118 100644 --- a/client/src/utils/uploadbox.js +++ b/client/src/utils/uploadbox.js @@ -6,61 +6,23 @@ import _ from "underscore"; import jQuery from "jquery"; import { getAppRoot } from "onload/loadConfig"; import * as tus from "tus-js-client"; +import axios from "axios"; + +function submitPayload(payload, cnf) { + axios + .post(`${getAppRoot()}api/tools/fetch`, payload) + .then((response) => { + cnf.success(response.data); + }) + .catch((error) => { + cnf.error(error.response.data.err_msg); + }); +} (($) => { // add event properties jQuery.event.props.push("dataTransfer"); - /** - xhr request helper - */ - var _uploadrequest = (config) => { - var cnf = $.extend( - { - error_default: "Please make sure the file is available.", - error_server: "Upload request failed.", - error_login: "Uploads require you to log in.", - error_retry: "Waiting for server to resume...", - }, - config - ); - console.debug(cnf); - var xhr = new XMLHttpRequest(); - xhr.open("POST", cnf.url, true); - xhr.setRequestHeader("Cache-Control", "no-cache"); - xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest"); - xhr.setRequestHeader("Accept", "application/json"); - xhr.onreadystatechange = () => { - if (xhr.readyState == xhr.DONE) { - if ([502, 0].indexOf(xhr.status) !== -1 && cnf.warning) { - cnf.warning(cnf.error_retry); - } else if (xhr.status < 200 || xhr.status > 299) { - var text = xhr.statusText; - if (xhr.status == 403) { - text = cnf.error_login; - } else if (xhr.status == 0) { - text = cnf.error_server; - } else if (!text) { - text = cnf.error_default; - } - cnf.error(`${text} (${xhr.status})`); - } else { - var response = null; - if (xhr.responseText) { - try { - response = jQuery.parseJSON(xhr.responseText); - } catch (e) { - response = xhr.responseText; - } - } - cnf.success(response); - } - } - }; - xhr.upload.addEventListener("progress", cnf.progress, false); - xhr.send(cnf.data); - }; - /** Posts chunked files to the API. */ @@ -91,13 +53,12 @@ import * as tus from "tus-js-client"; cnf.error(data.error_message); return; } - var file_data = data.files && data.files[0]; - if (!file_data) { - cnf.error(cnf.error_file); - return; + const file = data.files && data.files[0]; + if (!file) { + return submitPayload(data, cnf); } + console.log("file is", file); const startTime = performance.now(); - const file = file_data.file; const tusEndpoint = `${getAppRoot()}api/upload/resumable_upload/`; const chunkSize = cnf.chunkSize; console.debug(`Starting chunked uploads [size=${chunkSize}].`); @@ -121,25 +82,13 @@ import * as tus from "tus-js-client"; (performance.now() - startTime) / 1000 } seconds` ); - - const toolInputs = JSON.parse(data.payload.inputs); - toolInputs["files_0|file_data"] = { + // drop files from data + delete data["files"]; + data["files_0|file_data"] = { session_id: upload.url.split("/").at(-1), - name: file.name, + name: upload.file.name, }; - data.payload.inputs = JSON.stringify(toolInputs); - $.ajax({ - url: `${getAppRoot()}api/tools`, - method: "POST", - data: data.payload, - success: (tool_response) => { - cnf.success(tool_response); - }, - error: (tool_response) => { - var err_msg = tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg; - cnf.error(err_msg || cnf.error_tool); - }, - }); + submitPayload(data, cnf); }, }); // Check if there are any previous uploads to continue. @@ -155,98 +104,6 @@ import * as tus from "tus-js-client"; }); }; - /** - Posts multiple files without chunking to the API. - */ - $.uploadpost = function (config) { - var cnf = $.extend( - {}, - { - data: {}, - success: () => {}, - error: () => {}, - progress: () => {}, - url: null, - maxfilesize: 1048576 * 2048, - error_filesize: "File exceeds 2GB. Please use a FTP client.", - }, - config - ); - var data = cnf.data; - if (data.error_message) { - cnf.error(data.error_message); - return; - } - - // construct form data - var form = new FormData(); - for (const key in data.payload) { - form.append(key, data.payload[key]); - } - - // add files to submission - var sizes = 0; - for (const key in data.files) { - var d = data.files[key]; - form.append(d.name, d.file, d.file.name); - sizes += d.file.size; - } - - // check file size, unless it's an ftp file - if (sizes > cnf.maxfilesize) { - cnf.error(cnf.error_filesize); - return; - } - - // submit request - _uploadrequest({ - url: cnf.url, - data: form, - success: cnf.success, - error: cnf.error, - progress: (e) => { - if (e.lengthComputable) { - cnf.progress(Math.round((e.loaded * 100) / e.total)); - } - }, - }); - }; - /** - * - * @param {*} config - */ - $.datafetchpost = function (config) { - var cnf = $.extend( - {}, - { - data: {}, - success: () => {}, - error: () => {}, - progress: () => {}, - url: null, - }, - config - ); - var data = cnf.data; - if (data.error_message) { - cnf.error(data.error_message); - return; - } - - // submit request - _uploadrequest({ - url: cnf.url, - data: JSON.stringify(cnf.data), - success: cnf.success, - error: cnf.error, - progress: (e) => { - if (e.lengthComputable) { - cnf.progress(Math.round((e.loaded * 100) / e.total)); - } - }, - }); - }; - /** Handles the upload events drag/drop etc. */ @@ -398,24 +255,16 @@ import * as tus from "tus-js-client"; break; } - // get current file from queue - var file = queue[index]; - // remove from queue remove(index); // create and submit data - var submitter = $.uploadpost; + var submitter = $.uploadchunk; var requestData = opts.initialize(index); - if (file.chunk_mode && opts.chunkSize > 0) { - submitter = $.uploadchunk; - } else if (requestData.fetchRequest) { - submitter = $.datafetchpost; - } submitter({ url: opts.initUrl(index), - data: requestData.fetchRequest ? requestData.fetchRequest : requestData.uploadRequest, + data: requestData, success: (message) => { opts.success(index, message); process(); From c302aefbf6af9e569383075ee656cb1e0931da00 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 5 Nov 2021 19:01:01 +0100 Subject: [PATCH 010/781] Allow uploading multiple files via TUS in single fetch_data request That's needed for composite uploads via fetch_data. --- client/src/utils/uploadbox.js | 96 ++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/client/src/utils/uploadbox.js b/client/src/utils/uploadbox.js index a0671bd1118..c8ecde8fbb2 100644 --- a/client/src/utils/uploadbox.js +++ b/client/src/utils/uploadbox.js @@ -19,6 +19,53 @@ function submitPayload(payload, cnf) { }); } +function tusUpload(data, index, tusEndpoint, cnf) { + const startTime = performance.now(); + const chunkSize = cnf.chunkSize; + const file = data.files[index]; + if (!file) { + // We've uploaded all files, delete files from data and submit fetch payload + delete data["files"]; + return submitPayload(data, cnf); + } + console.debug(`Starting chunked upload for ${file.name} [chunkSize=${chunkSize}].`); + const upload = new tus.Upload(file, { + endpoint: tusEndpoint, + chunkSize: chunkSize, + metadata: data.payload, + onError: function (error) { + console.log("Failed because: " + error); + cnf.error(error); + }, + onProgress: function (bytesUploaded, bytesTotal) { + var percentage = ((bytesUploaded / bytesTotal) * 100).toFixed(2); + console.log(bytesUploaded, bytesTotal, percentage + "%"); + cnf.progress(percentage); + }, + onSuccess: function () { + console.log( + `Upload of ${upload.file.name} to ${upload.url} took ${(performance.now() - startTime) / 1000} seconds` + ); + data[`files_${index}|file_data`] = { + session_id: upload.url.split("/").at(-1), + name: upload.file.name, + }; + tusUpload(data, index + 1, tusEndpoint, cnf); + }, + }); + // Check if there are any previous uploads to continue. + upload.findPreviousUploads().then(function (previousUploads) { + // Found previous uploads so we select the first one. + if (previousUploads.length) { + console.log("previous Upload", previousUploads); + upload.resumeFromPreviousUpload(previousUploads[0]); + } + + // Start the upload + upload.start(); + }); +} + (($) => { // add event properties jQuery.event.props.push("dataTransfer"); @@ -53,55 +100,12 @@ function submitPayload(payload, cnf) { cnf.error(data.error_message); return; } - const file = data.files && data.files[0]; - if (!file) { + if (!data.files.length) { + // No files attached, don't need to use TUS uploader return submitPayload(data, cnf); } - console.log("file is", file); - const startTime = performance.now(); const tusEndpoint = `${getAppRoot()}api/upload/resumable_upload/`; - const chunkSize = cnf.chunkSize; - console.debug(`Starting chunked uploads [size=${chunkSize}].`); - - const upload = new tus.Upload(file, { - endpoint: tusEndpoint, - chunkSize: chunkSize, - metadata: data.payload, - onError: function (error) { - console.log("Failed because: " + error); - cnf.error(error); - }, - onProgress: function (bytesUploaded, bytesTotal) { - var percentage = ((bytesUploaded / bytesTotal) * 100).toFixed(2); - console.log(bytesUploaded, bytesTotal, percentage + "%"); - cnf.progress(percentage); - }, - onSuccess: function () { - console.log( - `Upload of ${upload.file.name} to ${upload.url} took ${ - (performance.now() - startTime) / 1000 - } seconds` - ); - // drop files from data - delete data["files"]; - data["files_0|file_data"] = { - session_id: upload.url.split("/").at(-1), - name: upload.file.name, - }; - submitPayload(data, cnf); - }, - }); - // Check if there are any previous uploads to continue. - upload.findPreviousUploads().then(function (previousUploads) { - // Found previous uploads so we select the first one. - if (previousUploads.length) { - console.log("previous Upload", previousUploads); - upload.resumeFromPreviousUpload(previousUploads[0]); - } - - // Start the upload - upload.start(); - }); + tusUpload(data, 0, tusEndpoint, cnf); }; /** From efbcdefa4a32f4826caab4dabb8ded6b61c54f4f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 6 Nov 2021 10:57:00 +0100 Subject: [PATCH 011/781] Add custom info field to fetch_data uploads --- lib/galaxy/tools/data_fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py index c0fa8227fe9..9f0dc707d49 100644 --- a/lib/galaxy/tools/data_fetch.py +++ b/lib/galaxy/tools/data_fetch.py @@ -295,7 +295,7 @@ def _fetch_target(upload_config, target): # Groom the dataset content if necessary datatype.groom_dataset_content(path) - rval = {"name": name, "filename": path, "dbkey": dbkey, "ext": ext, "link_data_only": link_data_only, "sources": sources, "hashes": hashes} + rval = {"name": name, "filename": path, "dbkey": dbkey, "ext": ext, "link_data_only": link_data_only, "sources": sources, "hashes": hashes, "info": f"uploaded {ext} file"} if staged_extra_files: rval["extra_files"] = os.path.abspath(staged_extra_files) return _copy_and_validate_simple_attributes(item, rval) From 0f3141f49ee0dfbc8f7c1d19dea1abea7d32c066 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 6 Nov 2021 11:22:10 +0100 Subject: [PATCH 012/781] Restore URL splitting --- client/src/components/Upload/helpers.js | 27 ++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/client/src/components/Upload/helpers.js b/client/src/components/Upload/helpers.js index 099bf93b6c3..7952345f536 100644 --- a/client/src/components/Upload/helpers.js +++ b/client/src/components/Upload/helpers.js @@ -20,10 +20,29 @@ export function uploadModelsToPayload(items, history_id, composite = false) { fileName = null; } const url = (item.get("file_uri") || item.get("file_path") || item.get("url_paste")).trim(); + const elem = { + dbkey: item.get("genome", "?"), + ext: item.get("extension", "auto"), + space_to_tab: item.get("space_to_tab"), + to_posix_lines: item.get("to_posix_lines"), + }; switch (item.get("file_mode")) { case "new": if (itemIsUrl(item)) { src = "url"; + /* Could be multiple URLs pasted in. + TODO: eliminate backbone models, + then suggest to split multiple URLs + across multiple uploads directly in upload modal, + instead of this intransparent magic. */ + return url.split("\n").map((splitUrl) => { + return { + src: src, + url: splitUrl, + name: fileName, + ...elem, + }; + }); } else { pasteContent = item.get("url_paste"); src = "pasted"; @@ -46,13 +65,11 @@ export function uploadModelsToPayload(items, history_id, composite = false) { url: url, paste_content: pasteContent, name: fileName, - dbkey: item.get("genome", "?"), - ext: item.get("extension", "auto"), - space_to_tab: item.get("space_to_tab"), - to_posix_lines: item.get("to_posix_lines"), + ...elem, }; }) - .filter((item) => item); + .filter((item) => item) + .flat(); const target = { destination: { type: "hdas" }, From e1c3e903747dcd39d01c1e1bed438a6635e44ec0 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 6 Nov 2021 13:12:51 +0100 Subject: [PATCH 013/781] Adjust test assertion --- lib/galaxy_test/selenium/test_history_dataset_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/selenium/test_history_dataset_state.py b/lib/galaxy_test/selenium/test_history_dataset_state.py index 92e07d2310d..09f96783413 100644 --- a/lib/galaxy_test/selenium/test_history_dataset_state.py +++ b/lib/galaxy_test/selenium/test_history_dataset_state.py @@ -12,7 +12,7 @@ BUTTON_TOOLTIPS = { "info": 'View details', "rerun": 'Run this job again', } -EXPECTED_TOOLHELP_TITLE_TEXT = 'Tool help for Upload File' +EXPECTED_TOOLHELP_TITLE_TEXT = 'Tool help for Data Fetch' TEST_DBKEY_TEXT = 'Honeybee (Apis mellifera): apiMel3 (apiMel3)' FIRST_HID = 1 From be73f7886e60710f664c8285825c8d8d630045ff Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 2 Nov 2021 12:12:53 -0400 Subject: [PATCH 014/781] Add hidden field to form builder component --- client/src/components/Form/FormElement.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue index ec565e59e02..ac85e579e69 100644 --- a/client/src/components/Form/FormElement.vue +++ b/client/src/components/Form/FormElement.vue @@ -22,6 +22,7 @@
+ Date: Tue, 2 Nov 2021 12:39:27 -0400 Subject: [PATCH 015/781] Add hidden field as Vue component --- client/src/components/Form/Elements/FormHidden.vue | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 client/src/components/Form/Elements/FormHidden.vue diff --git a/client/src/components/Form/Elements/FormHidden.vue b/client/src/components/Form/Elements/FormHidden.vue new file mode 100644 index 00000000000..2c2f606dfec --- /dev/null +++ b/client/src/components/Form/Elements/FormHidden.vue @@ -0,0 +1,13 @@ + + + From 46445486a12fb9b4c18dc466a4a6aeaca5a26348 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 2 Nov 2021 12:50:04 -0400 Subject: [PATCH 016/781] Remove backbone based hidden field from parameters --- .../components/Form/Elements/FormHidden.vue | 8 +++++++- .../components/Form/Elements/parameters.js | 20 ------------------- client/src/components/Form/FormElement.vue | 7 ++++++- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/client/src/components/Form/Elements/FormHidden.vue b/client/src/components/Form/Elements/FormHidden.vue index 2c2f606dfec..fb4d9879a5b 100644 --- a/client/src/components/Form/Elements/FormHidden.vue +++ b/client/src/components/Form/Elements/FormHidden.vue @@ -1,5 +1,7 @@ diff --git a/client/src/components/Form/Elements/parameters.js b/client/src/components/Form/Elements/parameters.js index 9be7ef1d0ee..3b6d7cecb24 100644 --- a/client/src/components/Form/Elements/parameters.js +++ b/client/src/components/Form/Elements/parameters.js @@ -26,13 +26,9 @@ export default Backbone.View.extend({ data_collection: "_fieldData", integer: "_fieldSlider", float: "_fieldSlider", - boolean: "_fieldBoolean", drill_down: "_fieldDrilldown", color: "_fieldColor", group_tag: "_fieldSelect", - hidden: "_fieldHidden", - hidden_data: "_fieldHidden", - baseurl: "_fieldHidden", library_data: "_fieldLibrary", ftpfile: "_fieldFtp", upload: "_fieldUpload", @@ -189,22 +185,6 @@ export default Backbone.View.extend({ }); }, - /** Hidden field */ - _fieldHidden: function (input_def) { - return new Ui.Hidden({ - id: `field-${input_def.id}`, - info: input_def.info, - }); - }, - - /** Boolean field */ - _fieldBoolean: function (input_def) { - return new Ui.Switch({ - id: `field-${input_def.id}`, - onchange: input_def.onchange, - }); - }, - /** Color picker field */ _fieldColor: function (input_def) { return new ColorPicker({ diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue index ac85e579e69..c2e0cf15f17 100644 --- a/client/src/components/Form/FormElement.vue +++ b/client/src/components/Form/FormElement.vue @@ -22,7 +22,12 @@
- + Date: Tue, 2 Nov 2021 12:52:08 -0400 Subject: [PATCH 017/781] Remove unused hidden field backbone wrapper --- client/src/mvc/ui/ui-misc.js | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/client/src/mvc/ui/ui-misc.js b/client/src/mvc/ui/ui-misc.js index 85c40eec3c9..e5a80ee9c6a 100644 --- a/client/src/mvc/ui/ui-misc.js +++ b/client/src/mvc/ui/ui-misc.js @@ -126,30 +126,6 @@ export var Input = Backbone.View.extend({ }, }); -/** Creates a hidden element input field used e.g. in the tool form */ -export var Hidden = Backbone.View.extend({ - initialize: function (options) { - this.model = (options && options.model) || new Backbone.Model(options); - this.setElement( - $("
") - .append((this.$info = $("
"))) - .append((this.$hidden = $("
"))) - ); - this.listenTo(this.model, "change", this.render, this); - this.render(); - }, - value: function (new_val) { - new_val !== undefined && this.model.set("value", new_val); - return this.model.get("value"); - }, - render: function () { - this.$el.attr("id", this.model.id); - this.$hidden.val(this.model.get("value")); - this.model.get("info") ? this.$info.show().text(this.model.get("info")) : this.$info.hide(); - return this; - }, -}); - export var NullableText = Backbone.View.extend({ initialize: function (options) { this.model = (options && options.model) || new Backbone.Model().set(options); @@ -316,7 +292,6 @@ export default { Select: Select, NullableText: NullableText, TextSelect: TextSelect, - Hidden: Hidden, Slider: Slider, Drilldown: Drilldown, }; From ef64878a6d89af3f6e8fd788f7413296dfaab397 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 9 Nov 2021 10:16:34 -0500 Subject: [PATCH 018/781] Replace qunit backbone tests with jest --- .../Form/Elements/FormHidden.test.js | 26 +++++++++++++++++++ client/tests/qunit/tests/ui_tests.js | 12 --------- 2 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 client/src/components/Form/Elements/FormHidden.test.js diff --git a/client/src/components/Form/Elements/FormHidden.test.js b/client/src/components/Form/Elements/FormHidden.test.js new file mode 100644 index 00000000000..c25c4e382e7 --- /dev/null +++ b/client/src/components/Form/Elements/FormHidden.test.js @@ -0,0 +1,26 @@ +import { mount } from "@vue/test-utils"; +import { getLocalVue } from "jest/helpers"; +import FormHidden from "./FormHidden"; + +const localVue = getLocalVue(); + +describe("FormHidden", () => { + let wrapper; + + beforeEach(() => { + wrapper = mount(FormHidden, { + propsData: { + value: false, + info: "info", + }, + localVue, + }); + }); + + it("check initial value and value change", async () => { + expect(wrapper.vm.value).toBe(false); + await wrapper.setProps({ value: true }); + expect(wrapper.vm.value).toBe(true); + expect(wrapper.text()).toBe("info"); + }); +}); diff --git a/client/tests/qunit/tests/ui_tests.js b/client/tests/qunit/tests/ui_tests.js index 706f4eb2d24..a869bfccfbf 100644 --- a/client/tests/qunit/tests/ui_tests.js +++ b/client/tests/qunit/tests/ui_tests.js @@ -713,18 +713,6 @@ QUnit.test("message", function (assert) { assert.ok(message.$el.html() === "_new_message", "Correct new message."); }); -QUnit.test("hidden", function (assert) { - var hidden = new Ui.Hidden(); - $("body").prepend(hidden.$el); - hidden.model.set("info", "_info"); - assert.ok(hidden.$info.css("display", "block"), "Info shown."); - assert.ok(hidden.$info.html() === "_info", "Info text correct."); - hidden.model.set("info", ""); - assert.ok(hidden.$info.css("display", "none"), "Info hidden."); - hidden.model.set("value", "_value"); - assert.ok(hidden.$hidden.val() === "_value", "Correct value"); -}); - QUnit.test("select-content", function (assert) { var select = new SelectContent.View({}); $("body").prepend(select.$el); From e0054a3f657f6050fe811bf3ad955447e5024078 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 9 Nov 2021 16:19:26 +0100 Subject: [PATCH 019/781] create sniffer for GRO file type --- .../config/sample/datatypes_conf.xml.sample | 3 +- lib/galaxy/datatypes/molecules.py | 39 +++++++++++++++ lib/galaxy/datatypes/test/5e5z.gro | 50 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/datatypes/test/5e5z.gro diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index d5d81f24b6e..ca09840e716 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -751,7 +751,7 @@ - + @@ -1035,6 +1035,7 @@ + diff --git a/lib/galaxy/datatypes/molecules.py b/lib/galaxy/datatypes/molecules.py index 15b5c485ebc..663f4825a2e 100644 --- a/lib/galaxy/datatypes/molecules.py +++ b/lib/galaxy/datatypes/molecules.py @@ -939,3 +939,42 @@ class CML(GenericXml): if molecule_found: out.write(line) out.write("\n") + + +class GRO(GenericMolFile): + """ + GROMACS structure format. + https://manual.gromacs.org/current/reference-manual/file-formats.html#gro + """ + file_ext = "gro" + + def sniff_prefix(self, file_prefix): + """ + Try to guess if the file is a GRO file. + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('5e5z.gro') + >>> GRO().sniff_prefix(fname) + True + >>> fname = get_test_fname('5e5z.pdb') + >>> GRO().sniff_prefix(fname) + False + """ + headers = get_headers(file_prefix, sep='\n', count=300) + try: + int(headers[1][0]) # the second line should just be the number of atoms + except ValueError: + return False + for line in headers[2:-1]: # skip the first, second and last lines + if not re.search(r'^[0-9 ]{5}[a-zA-Z0-9 ]{10}[0-9 ]{5}[0-9 -]{4}\.[0-9]{3}[0-9 -]{4}\.[0-9]{3}[0-9 -]{4}\.[0-9]{3}', line[0]): + return False + return True + + def set_peek(self, dataset, is_multi_byte=False): + if not dataset.dataset.purged: + dataset.peek = get_file_peek(dataset.file_name) + atom_number = int(dataset.peek.split('\n')[1]) + dataset.blurb = f"{atom_number} atoms" + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' diff --git a/lib/galaxy/datatypes/test/5e5z.gro b/lib/galaxy/datatypes/test/5e5z.gro new file mode 100644 index 00000000000..7856099921f --- /dev/null +++ b/lib/galaxy/datatypes/test/5e5z.gro @@ -0,0 +1,50 @@ +LVHSSN (RESIDUES 16-21) FROM ISLET AMYLOID POLYPEPTIDE + 47 + 1LEU N 1 2.045 1.847 0.927 + 1LEU CA 2 1.954 1.875 1.038 + 1LEU C 3 2.006 1.813 1.167 + 1LEU O 4 2.043 1.696 1.170 + 1LEU CB 5 1.813 1.822 1.006 + 1LEU CG 6 1.697 1.842 1.105 + 1LEU CD1 7 1.713 1.759 1.232 + 1LEU CD2 8 1.678 1.990 1.139 + 2VAL N 9 2.009 1.894 1.273 + 2VAL CA 10 2.034 1.842 1.406 + 2VAL C 11 1.935 1.906 1.502 + 2VAL O 12 1.912 2.026 1.498 + 2VAL CB 13 2.177 1.875 1.455 + 2VAL CG1 14 2.194 1.835 1.602 + 2VAL CG2 15 2.280 1.805 1.368 + 3HIS N 16 1.874 1.824 1.587 + 3HIS CA 17 1.798 1.876 1.699 + 3HIS C 18 1.838 1.810 1.830 + 3HIS O 19 1.858 1.689 1.836 + 3HIS CB 20 1.647 1.860 1.680 + 3HIS CG 21 1.570 1.885 1.806 + 3HIS ND1 22 1.532 1.784 1.892 + 3HIS CD2 23 1.530 2.000 1.864 + 3HIS CE1 24 1.470 1.836 1.996 + 3HIS NE2 25 1.466 1.967 1.982 + 4SER N 26 1.847 1.892 1.935 + 4SER CA 27 1.873 1.844 2.071 + 4SER C 28 1.819 1.946 2.170 + 4SER O 29 1.838 2.066 2.153 + 4SER CB 30 2.023 1.826 2.093 + 4SER OG 31 2.051 1.800 2.230 + 5SER N 32 1.751 1.899 2.275 + 5SER CA 33 1.703 1.989 2.379 + 5SER C 34 1.805 2.009 2.491 + 5SER O 35 1.779 2.080 2.587 + 5SER CB 36 1.569 1.941 2.437 + 5SER OG 37 1.585 1.823 2.514 + 6ASN N 38 1.921 1.945 2.478 + 6ASN CA 39 2.029 1.961 2.576 + 6ASN C 40 2.095 2.098 2.566 + 6ASN O 41 2.171 2.137 2.654 + 6ASN CB 42 2.136 1.853 2.561 + 6ASN CG 43 2.097 1.722 2.626 + 6ASN OD1 44 1.995 1.713 2.695 + 6ASN ND2 45 2.178 1.619 2.605 + 6ASN OXT 46 2.076 2.171 2.469 + 101HOH O 47 2.258 1.983 1.046 + 3.77337 3.77337 3.77337 From a7efc133cd5a8b4f510ffbee0dc175637b7712e0 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Wed, 6 Oct 2021 12:12:31 -0400 Subject: [PATCH 020/781] Add datatype definition for AGP assemblies This adds support for the "A Golden Path" 2.0 assembly file format developed by UCSC --- .../config/sample/datatypes_conf.xml.sample | 2 + lib/galaxy/datatypes/tabular.py | 50 +++++ lib/galaxy/datatypes/test/eg1.agp | 174 ++++++++++++++++++ lib/galaxy/datatypes/test/eg2.agp | 56 ++++++ 4 files changed, 282 insertions(+) create mode 100644 lib/galaxy/datatypes/test/eg1.agp create mode 100644 lib/galaxy/datatypes/test/eg2.agp diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index bcf16e1cd30..ce0068f51a2 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -4,6 +4,7 @@ + @@ -1102,6 +1103,7 @@ module is fixed to not read the entire file. --> + diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 2716e1ceea3..5789ff8375e 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -460,6 +460,56 @@ class Taxonomy(Tabular): return self.make_html_table(dataset, column_names=self.column_names) +@build_sniff_from_prefix +class GoldenPath(Tabular): + """Class describing UCSC's Golden Path assembly format""" + edam_format = 'format_3693' + file_ext = 'agp' + + def sniff_prefix(self, file_prefix): + """ + Checks for and does cursory validation on data that looks like AGP + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('eg1.agp') + >>> GoldenPath().sniff(fname) + True + >>> fname = get_test_fname('eg2.agp') + >>> GoldenPath().sniff(fname) + True + >>> fname = get_test_fname('1.bed') + >>> GoldenPath().sniff(fname) + False + >>> fname = get_test_fname('2.tabular') + >>> GoldenPath().sniff(fname) + False + """ + found_non_comment_lines = False + # try: + if True: + for line in iter_headers(file_prefix, '\t', comment_designator='#'): + if line and not line[0].startswith('#'): + if len(line) != 9: + return False + assert line[4] in ['A', 'D', 'F', 'G', 'O', 'P', 'W', 'N', 'U'] + ostensible_numbers = line[1:3] + if line[4] in ['U', 'N']: + ostensible_numbers.append(line[5]) + assert line[6] in ['scaffold', 'contig', 'centromere', 'short_arm', 'heterochromatin', 'telomere', 'repeat'] + assert line[7] in ['yes', 'no'] + assert line[8] in ['na', 'paired-ends', 'align_genus', 'align_xgenus', 'align_trnscript', 'within_clone', 'clone_contig', 'map', 'strobe', 'unspecified'] + else: + ostensible_numbers.extend([line[6], line[7]]) + assert line[8] in ['+', '-', '?', '0', 'na'] + if line[4] == 'U': + assert int(line[5]) == 100 + assert all(map(lambda x: str(x).isnumeric() and int(x) > 0, ostensible_numbers)) + found_non_comment_lines = True + # except Exception: + # return False + return found_non_comment_lines + + @dataproviders.decorators.has_dataproviders @build_sniff_from_prefix class Sam(Tabular): diff --git a/lib/galaxy/datatypes/test/eg1.agp b/lib/galaxy/datatypes/test/eg1.agp new file mode 100644 index 00000000000..73650c96370 --- /dev/null +++ b/lib/galaxy/datatypes/test/eg1.agp @@ -0,0 +1,174 @@ +##agp-version 2.0 +# ORGANISM: Homo sapiens +# TAX_ID: 9606 +# ASSEMBLY NAME: EG1 +# ASSEMBLY DATE: 09-November-2011 +# GENOME CENTER: NCBI +# DESCRIPTION: Example AGP specifying the assembly of scaffolds from WGS contigs +EG1_scaffold1 1 3043 1 W AADB02037551.1 1 3043 + +EG1_scaffold2 1 40448 1 W AADB02037552.1 1 40448 + +EG1_scaffold2 40449 40661 2 N 213 scaffold yes paired-ends +EG1_scaffold2 40662 117642 3 W AADB02037553.1 1 76981 + +EG1_scaffold2 117643 117718 4 N 76 scaffold yes paired-ends +EG1_scaffold2 117719 145387 5 W AADB02037554.1 1 27669 + +EG1_scaffold2 145388 145485 6 N 98 scaffold yes paired-ends +EG1_scaffold2 145486 148437 7 W AADB02037555.1 1 2952 + +EG1_scaffold2 148438 148560 8 N 123 scaffold yes paired-ends +EG1_scaffold2 148561 152709 9 W AADB02037556.1 1 4149 - +EG1_scaffold2 152710 153074 10 N 365 scaffold yes paired-ends +EG1_scaffold2 153075 158982 11 W AADB02037557.1 1 5908 + +EG1_scaffold2 158983 163333 12 N 4351 scaffold yes paired-ends +EG1_scaffold2 163334 172851 13 W AADB02037558.1 1 9518 + +EG1_scaffold2 172852 172894 14 N 43 scaffold yes paired-ends +EG1_scaffold2 172895 213547 15 W AADB02037559.1 1 40653 + +EG1_scaffold2 213548 213664 16 N 117 scaffold yes paired-ends +EG1_scaffold2 213665 226801 17 W AADB02037560.1 1 13137 + +EG1_scaffold2 226802 227200 18 N 399 scaffold yes paired-ends +EG1_scaffold2 227201 230202 19 W AADB02037561.1 1 3002 + +EG1_scaffold2 230203 230907 20 N 705 scaffold yes paired-ends +EG1_scaffold2 230908 232083 21 W AADB02037562.1 1 1176 - +EG1_scaffold2 232084 232808 22 N 725 scaffold yes paired-ends +EG1_scaffold2 232809 234010 23 W AADB02037563.1 1 1202 + +EG1_scaffold2 234011 234049 24 N 39 scaffold yes paired-ends +EG1_scaffold2 234050 235260 25 W AADB02037564.1 1 1211 + +EG1_scaffold2 235261 248408 26 N 13148 scaffold yes paired-ends +EG1_scaffold2 248409 1659882 27 W AADB02037565.1 1 1411474 + +EG1_scaffold2 1659883 1679401 28 N 19519 scaffold yes paired-ends +EG1_scaffold2 1679402 1683501 29 W AADB02037566.1 1 4100 + +EG1_scaffold2 1683502 1683540 30 N 39 scaffold yes paired-ends +EG1_scaffold2 1683541 1688624 31 W AADB02037567.1 1 5084 + +EG1_scaffold2 1688625 1691197 32 N 2573 scaffold yes paired-ends +EG1_scaffold2 1691198 1710847 33 W AADB02037568.1 1 19650 + +EG1_scaffold2 1710848 1719842 34 N 8995 scaffold yes paired-ends +EG1_scaffold2 1719843 1799353 35 W AADB02037569.1 1 79511 + +EG1_scaffold2 1799354 1799652 36 N 299 scaffold yes paired-ends +EG1_scaffold2 1799653 2333358 37 W AADB02037570.1 1 533706 + +EG1_scaffold2 2333359 2333512 38 N 154 scaffold yes paired-ends +EG1_scaffold2 2333513 2537251 39 W AADB02037571.1 1 203739 + +EG1_scaffold2 2537252 2561289 40 N 24038 scaffold yes paired-ends +EG1_scaffold2 2561290 2741173 41 W AADB02037572.1 1 179884 + +EG1_scaffold2 2741174 2741349 42 N 176 scaffold yes paired-ends +EG1_scaffold2 2741350 2744341 43 W AADB02037573.1 1 2992 + +EG1_scaffold2 2744342 2744661 44 N 320 scaffold yes paired-ends +EG1_scaffold2 2744662 2764215 45 W AADB02037574.1 1 19554 + +EG1_scaffold2 2764216 2764449 46 N 234 scaffold yes paired-ends +EG1_scaffold2 2764450 2771367 47 W AADB02037575.1 1 6918 + +EG1_scaffold2 2771368 2774211 48 N 2844 scaffold yes paired-ends +EG1_scaffold2 2774212 2920720 49 W AADB02037576.1 1 146509 + +EG1_scaffold2 2920721 2920803 50 N 83 scaffold yes paired-ends +EG1_scaffold2 2920804 3250065 51 W AADB02037577.1 1 329262 + +EG1_scaffold2 3250066 3250318 52 N 253 scaffold yes paired-ends +EG1_scaffold2 3250319 3292615 53 W AADB02037578.1 1 42297 + +EG1_scaffold2 3292616 3292783 54 N 168 scaffold yes paired-ends +EG1_scaffold2 3292784 3299108 55 W AADB02037579.1 1 6325 + +EG1_scaffold2 3299109 3300377 56 N 1269 scaffold yes paired-ends +EG1_scaffold2 3300378 3371051 57 W AADB02037580.1 1 70674 + +EG1_scaffold3 1 92327 1 W AADB02037581.1 1 92327 + +EG1_scaffold4 1 7150 1 W AADB02037582.1 1 7150 + +EG1_scaffold5 1 5578 1 W AADB02037583.1 1 5578 + +EG1_scaffold6 1 4590 1 W AADB02037584.1 1 4590 + +EG1_scaffold6 4591 4781 2 N 191 scaffold yes paired-ends +EG1_scaffold6 4782 77946 3 W AADB02037585.1 1 73165 + +EG1_scaffold7 1 1021537 1 W AADB02037586.1 1 1021537 + +EG1_scaffold7 1021538 1028000 2 N 6463 scaffold yes paired-ends +EG1_scaffold7 1028001 1427727 3 W AADB02037587.1 1 399727 + +EG1_scaffold7 1427728 1427918 4 N 191 scaffold yes paired-ends +EG1_scaffold7 1427919 1443997 5 W AADB02037588.1 1 16079 + +EG1_scaffold7 1443998 1473572 6 N 29575 scaffold yes paired-ends +EG1_scaffold7 1473573 1478879 7 W AADB02037589.1 1 5307 + +EG1_scaffold7 1478880 1485983 8 N 7104 scaffold yes paired-ends +EG1_scaffold7 1485984 1490402 9 W AADB02037590.1 1 4419 + +EG1_scaffold7 1490403 1519302 10 N 28900 scaffold yes paired-ends +EG1_scaffold7 1519303 1523152 11 W AADB02037591.1 1 3850 - +EG1_scaffold7 1523153 1523208 12 N 56 scaffold yes paired-ends +EG1_scaffold7 1523209 1542142 13 W AADB02037592.1 1 18934 + +EG1_scaffold7 1542143 1542817 14 N 675 scaffold yes paired-ends +EG1_scaffold7 1542818 1548665 15 W AADB02037593.1 1 5848 + +EG1_scaffold7 1548666 1575049 16 N 26384 scaffold yes paired-ends +EG1_scaffold7 1575050 1596422 17 W AADB02037594.1 1 21373 + +EG1_scaffold7 1596423 1596731 18 N 309 scaffold yes paired-ends +EG1_scaffold7 1596732 1599823 19 W AADB02037595.1 1 3092 + +EG1_scaffold8 1 1925 1 W AADB02037596.1 1 1925 - +EG1_scaffold8 1926 9512 2 N 7587 scaffold yes paired-ends +EG1_scaffold8 9513 285397 3 W AADB02037597.1 1 275885 + +EG1_scaffold8 285398 285563 4 N 166 scaffold yes paired-ends +EG1_scaffold8 285564 1072950 5 W AADB02037598.1 1 787387 + +EG1_scaffold8 1072951 1073157 6 N 207 scaffold yes paired-ends +EG1_scaffold8 1073158 1428275 7 W AADB02037599.1 1 355118 + +EG1_scaffold9 1 5125 1 W AADB02037600.1 1 5125 + +EG1_scaffold9 5126 13025 2 N 7900 scaffold yes paired-ends +EG1_scaffold9 13026 16059 3 W AADB02037601.1 1 3034 + +EG1_scaffold10 1 2750 1 W AADB02037602.1 1 2750 + +EG1_scaffold11 1 4269 1 W AADB02037603.1 1 4269 + +EG1_scaffold12 1 123770 1 W AADB02037604.1 1 123770 + +EG1_scaffold12 123771 125913 2 N 2143 scaffold yes paired-ends +EG1_scaffold12 125914 673965 3 W AADB02037605.1 1 548052 + +EG1_scaffold12 673966 674054 4 N 89 scaffold yes paired-ends +EG1_scaffold12 674055 704944 5 W AADB02037606.1 1 30890 - +EG1_scaffold12 704945 705106 6 N 162 scaffold yes paired-ends +EG1_scaffold12 705107 1128554 7 W AADB02037607.1 1 423448 - +EG1_scaffold13 1 72929 1 W AADB02037608.1 1 72929 + +EG1_scaffold13 72930 77168 2 N 4239 scaffold yes paired-ends +EG1_scaffold13 77169 95070 3 W AADB02037609.1 1 17902 + +EG1_scaffold14 1 1516 1 W AADB02037610.1 1 1516 + +EG1_scaffold14 1517 1937 2 N 421 scaffold yes paired-ends +EG1_scaffold14 1938 8159 3 W AADB02037611.1 1 6222 - +EG1_scaffold14 8160 10712 4 N 2553 scaffold yes paired-ends +EG1_scaffold14 10713 13240 5 W AADB02037612.1 1 2528 - +EG1_scaffold14 13241 13376 6 N 136 scaffold yes paired-ends +EG1_scaffold14 13377 15189 7 W AADB02037613.1 1 1813 + +EG1_scaffold14 15190 15334 8 N 145 scaffold yes paired-ends +EG1_scaffold14 15335 16645 9 W AADB02037614.1 1 1311 + +EG1_scaffold14 16646 16885 10 N 240 scaffold yes paired-ends +EG1_scaffold14 16886 18205 11 W AADB02037615.1 1 1320 - +EG1_scaffold14 18206 18512 12 N 307 scaffold yes paired-ends +EG1_scaffold14 18513 22267 13 W AADB02037616.1 1 3755 - +EG1_scaffold14 22268 32736 14 N 10469 scaffold yes paired-ends +EG1_scaffold14 32737 89286 15 W AADB02037617.1 1 56550 - +EG1_scaffold14 89287 95626 16 N 6340 scaffold yes paired-ends +EG1_scaffold14 95627 100806 17 W AADB02037618.1 1 5180 + +EG1_scaffold15 1 379475 1 W AADB02037619.1 1 379475 + +EG1_scaffold16 1 203038 1 W AADB02037620.1 1 203038 + +EG1_scaffold16 203039 207736 2 N 4698 scaffold yes paired-ends +EG1_scaffold16 207737 215711 3 W AADB02037621.1 1 7975 - +EG1_scaffold16 215712 222719 4 N 7008 scaffold yes paired-ends +EG1_scaffold16 222720 225744 5 W AADB02037622.1 1 3025 + +EG1_scaffold16 225745 226124 6 N 380 scaffold yes paired-ends +EG1_scaffold16 226125 229149 7 W AADB02037623.1 1 3025 + +EG1_scaffold16 229150 233378 8 N 4229 scaffold yes paired-ends +EG1_scaffold16 233379 259169 9 W AADB02037624.1 1 25791 + +EG1_scaffold16 259170 259316 10 N 147 scaffold yes paired-ends +EG1_scaffold16 259317 265416 11 W AADB02037625.1 1 6100 + +EG1_scaffold16 265417 265579 12 N 163 scaffold yes paired-ends +EG1_scaffold16 265580 284551 13 W AADB02037626.1 1 18972 + +EG1_scaffold16 284552 286290 14 N 1739 scaffold yes paired-ends +EG1_scaffold16 286291 290140 15 W AADB02037627.1 1 3850 + +EG1_scaffold16 290141 290881 16 N 741 scaffold yes paired-ends +EG1_scaffold16 290882 323606 17 W AADB02037628.1 1 32725 + +EG1_scaffold16 323607 325827 18 N 2221 scaffold yes paired-ends +EG1_scaffold16 325828 332848 19 W AADB02037629.1 1 7021 + +EG1_scaffold16 332849 335762 20 N 2914 scaffold yes paired-ends +EG1_scaffold16 335763 343737 21 W AADB02037630.1 1 7975 + +EG1_scaffold16 343738 367895 22 N 24158 scaffold yes paired-ends +EG1_scaffold16 367896 429363 23 W AADB02037631.1 1 61468 + +EG1_scaffold16 429364 429716 24 N 353 scaffold yes paired-ends +EG1_scaffold16 429717 537528 25 W AADB02037632.1 1 107812 + +EG1_scaffold17 1 29593 1 W AADB02037633.1 1 29593 + +EG1_scaffold18 1 5057 1 W AADB02037634.1 1 5057 + +EG1_scaffold19 1 307775 1 W AADB02037635.1 1 307775 + +EG1_scaffold20 1 16023 1 W AADB02037636.1 1 16023 + +EG1_scaffold21 1 94670 1 W AADB02037637.1 1 94670 + +EG1_scaffold21 94671 105789 2 N 11119 scaffold yes paired-ends +EG1_scaffold21 105790 139901 3 W AADB02037638.1 1 34112 + +EG1_scaffold21 139902 141260 4 N 1359 scaffold yes paired-ends +EG1_scaffold21 141261 242057 5 W AADB02037639.1 1 100797 + +EG1_scaffold21 242058 258444 6 N 16387 scaffold yes paired-ends +EG1_scaffold21 258445 262063 7 W AADB02037640.1 1 3619 - +EG1_scaffold21 262064 262266 8 N 203 scaffold yes paired-ends +EG1_scaffold21 262267 265059 9 W AADB02037641.1 1 2793 - +EG1_scaffold21 265060 272140 10 N 7081 scaffold yes paired-ends +EG1_scaffold21 272141 342501 11 W AADB02037642.1 1 70361 + +EG1_scaffold21 342502 343797 12 N 1296 scaffold yes paired-ends +EG1_scaffold21 343798 656420 13 W AADB02037643.1 1 312623 + +EG1_scaffold22 1 4051 1 W AADB02037644.1 1 4051 + +EG1_scaffold23 1 4430 1 W AADB02037645.1 1 4430 + diff --git a/lib/galaxy/datatypes/test/eg2.agp b/lib/galaxy/datatypes/test/eg2.agp new file mode 100644 index 00000000000..57dea216b7a --- /dev/null +++ b/lib/galaxy/datatypes/test/eg2.agp @@ -0,0 +1,56 @@ +##agp-version 2.0 +# ORGANISM: Homo sapiens +# TAX_ID: 9606 +# ASSEMBLY NAME: EG1 +# ASSEMBLY DATE: 09-November-2011 +# GENOME CENTER: NCBI +# DESCRIPTION: Example AGP specifying the assembly of chromosome Y from WGS scaffolds +# COMMENTS: +# Three scaffolds are placed but have unknown orientation. +chrY 1 10000 1 N 10000 telomere no na +chrY 10001 13043 2 W EG1_scaffold1 1 3043 ? +chrY 13044 63043 3 N 50000 contig no na +chrY 63044 3434094 4 W EG1_scaffold2 1 3371051 + +chrY 3434095 3484094 5 N 50000 contig no na +chrY 3484095 3576421 6 W EG1_scaffold3 1 92327 + +chrY 3576422 3626421 7 N 50000 contig no na +chrY 3626422 3633571 8 W EG1_scaffold4 1 7150 + +chrY 3633572 3683571 9 N 50000 contig no na +chrY 3683572 3689149 10 W EG1_scaffold5 1 5578 + +chrY 3689150 3739149 11 N 50000 contig no na +chrY 3739150 3817095 12 W EG1_scaffold6 1 77946 + +chrY 3817096 3867095 13 N 50000 contig no na +chrY 3867096 5466918 14 W EG1_scaffold7 1 1599823 + +chrY 5466919 5516918 15 N 50000 contig no na +chrY 5516919 6945193 16 W EG1_scaffold8 1 1428275 + +chrY 6945194 6995193 17 N 50000 contig no na +chrY 6995194 7011252 18 W EG1_scaffold9 1 16059 + +chrY 7011253 7061252 19 N 50000 contig no na +chrY 7061253 7064002 20 W EG1_scaffold10 1 2750 ? +chrY 7064003 8064002 21 N 1000000 centromere no na +chrY 8064003 8068271 22 W EG1_scaffold11 1 4269 + +chrY 8068272 8118271 23 N 50000 contig no na +chrY 8118272 9246825 24 W EG1_scaffold12 1 1128554 + +chrY 9246826 9296825 25 N 50000 contig no na +chrY 9296826 9391895 26 W EG1_scaffold13 1 95070 + +chrY 9391896 9441895 27 N 50000 contig no na +chrY 9441896 9542701 28 W EG1_scaffold14 1 100806 + +chrY 9542702 9592701 29 N 50000 contig no na +chrY 9592702 9972176 30 W EG1_scaffold15 1 379475 + +chrY 9972177 10022176 31 N 50000 contig no na +chrY 10022177 10559704 32 W EG1_scaffold16 1 537528 + +chrY 10559705 10609704 33 N 50000 contig no na +chrY 10609705 10639297 34 W EG1_scaffold17 1 29593 + +chrY 10639298 10689297 35 N 50000 contig no na +chrY 10689298 10694354 36 W EG1_scaffold18 1 5057 ? +chrY 10694355 10744354 37 N 50000 contig no na +chrY 10744355 11052129 38 W EG1_scaffold19 1 307775 + +chrY 11052130 11102129 39 N 50000 contig no na +chrY 11102130 11118152 40 W EG1_scaffold20 1 16023 + +chrY 11118153 11168152 41 N 50000 contig no na +chrY 11168153 11824572 42 W EG1_scaffold21 1 656420 + +chrY 11824573 11874572 43 N 50000 contig no na +chrY 11874573 11878623 44 W EG1_scaffold22 1 4051 + +chrY 11878624 11928623 45 N 50000 contig no na +chrY 11928624 11933053 46 W EG1_scaffold23 1 4430 + +chrY 11933054 11943053 47 N 10000 telomere no na From 0be337640beb3c42fe7811ca2e2b22dd86ff4077 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Wed, 6 Oct 2021 13:02:07 -0400 Subject: [PATCH 021/781] Cite correct organization. --- lib/galaxy/datatypes/tabular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 5789ff8375e..fa96e49c8d5 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -462,7 +462,7 @@ class Taxonomy(Tabular): @build_sniff_from_prefix class GoldenPath(Tabular): - """Class describing UCSC's Golden Path assembly format""" + """Class describing NCBI's Golden Path assembly format""" edam_format = 'format_3693' file_ext = 'agp' From 0b159dd43f685815841926ffa9d425bad8d5d9a0 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Wed, 6 Oct 2021 15:05:42 -0400 Subject: [PATCH 022/781] Restore flow control. --- lib/galaxy/datatypes/tabular.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index fa96e49c8d5..360e60afc2a 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -485,8 +485,7 @@ class GoldenPath(Tabular): False """ found_non_comment_lines = False - # try: - if True: + try: for line in iter_headers(file_prefix, '\t', comment_designator='#'): if line and not line[0].startswith('#'): if len(line) != 9: @@ -505,8 +504,8 @@ class GoldenPath(Tabular): assert int(line[5]) == 100 assert all(map(lambda x: str(x).isnumeric() and int(x) > 0, ostensible_numbers)) found_non_comment_lines = True - # except Exception: - # return False + except Exception: + return False return found_non_comment_lines From 79f89e0bd8b6da3c6def550ce11f403569e7c8b3 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Fri, 8 Oct 2021 12:50:55 -0400 Subject: [PATCH 023/781] Fix upload datatype test, add data provider. --- .../config/sample/datatypes_conf.xml.sample | 2 +- lib/galaxy/datatypes/dataproviders/column.py | 18 + lib/galaxy/datatypes/goldenpath.py | 600 ++++++++++++++++++ lib/galaxy/datatypes/tabular.py | 5 + .../visualization/data_providers/genome.py | 22 + .../visualization/data_providers/registry.py | 10 +- 6 files changed, 655 insertions(+), 2 deletions(-) create mode 100755 lib/galaxy/datatypes/goldenpath.py diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index ce0068f51a2..00d36c8fbe2 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -892,6 +892,7 @@ + @@ -1103,7 +1104,6 @@ module is fixed to not read the entire file. --> - diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index 6a94597bcf4..d03c1f30809 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -6,6 +6,8 @@ import logging import re from urllib.parse import unquote_plus +from galaxy.datatypes.goldenpath import AGPFile + from . import line _TODO = """ @@ -353,3 +355,19 @@ class DictDataProvider(ColumnarDataProvider): for column_values in parent_gen: map = dict(zip(self.column_names, column_values)) yield map + + +class GoldenPathDataProvider(DictDataProvider): + ''' + Class that defines a data provider for AGP datasets. + ''' + settings = {} + + def __init__(self, source, **kwargs): + self.goldenpath = AGPFile(source) + self.column_names = [str(gp) for gp in self.goldenpath] + super().__init__(source, **kwargs) + + def __iter__(self): + for gp in self.goldenpath: + yield gp diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py new file mode 100755 index 00000000000..c735239186d --- /dev/null +++ b/lib/galaxy/datatypes/goldenpath.py @@ -0,0 +1,600 @@ +""" +MIT License + +Copyright (c) 2021 Michael Alonge + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import os +import abc + + +class AGPError(Exception): + """ Exception raised for AGP related errors. """ + + def __init__(self, fname, line_number, message="Error in AGP file."): + self.fname = fname + self.line_number = line_number + self.message = message + + self.report = "" + self.report += "\n\nFILE : {}\n".format(self.fname) + self.report += "LINE : {}\n".format(self.line_number) + self.report += "ERROR: {}\n".format(self.message) + super().__init__(self.report) + + def __repr__(self): + return "AGPError" + + +class AGPFile: + + """ + A class storing the contents of an AGP v2.1 file. https://www.ncbi.nlm.nih.gov/assembly/agp/AGP_Specification/ + + The class is able to read new AGP lines in order to sequentially build the complete file. + + The class should be capable of checking the validity of the file, as well as writing the AGP contents + to a file stream. + + Common abbreviations: + "comp": AGP component + "obj": AGP object + "pid": AGP part number + """ + + def __init__(self, in_file, mode="r"): + if mode != "r" and mode != "w": + raise ValueError("AGPFile mode must be either read ('r') or write ('w').") + + self._agp_version = "2.1" + self._fname = os.path.abspath(in_file) + + # Store comment and AGP lines separately. + self._comment_lines = [] + self._objects = [] + + # Store info enabling us to keep track of the state of the AGP file + self._current_obj = None + self._seen_objs = set() + + # Read the contents of the AGP file + if mode == "r": + self._read_file() + + def __iter__(self): + for obj in self._objects: + yield str(obj.obj), [dict(agp_line) for agp_line in obj] + + def _read_file(self): + """ + Read the agp file associated with this instance of the class. If that file doesn't exist yet, + proceed without reading anything. + + When reading, check the validity of individual AGP lines. + """ + if not os.path.isfile(self.fname): + return + + # The AGP file exists. Initialize everything. + self._comment_lines = [] + self._objects = [] + self._current_obj = None + self._seen_objs = set() + + line_number = 0 + in_body = False + with open(self.fname, "r") as f: + for line in f: + line_number += 1 + line = line.rstrip("\n") + if line.startswith("#"): + if not in_body: + self._comment_lines.append(line) + else: + raise AGPError(self.fname, line_number, "illegal comment in AGP body") + continue + + # In a valid AGP file, we should no longer see comment lines + in_body = True + fields = line.split("\t") + + # There should be exactly 9 tab delimited fields + if not len(fields) == 9: + raise AGPError(self.fname, line_number, "detected more than 9 tab delimited fields") + + # All fields should have a value + if not all(fields): + raise AGPError(self.fname, line_number, "detected an empty field") + + # Instantiate all the AGPLine objects. These will do line-specific validations. + if fields[4] == "N" or fields[4] == "U": + agp_line = AGPGapLine(self.fname, line_number, *fields) + else: + agp_line = AGPSeqLine(self.fname, line_number, *fields) + + self._add_line(agp_line) + + def _add_line(self, agp_line): + # Perform validity checks if this is a new object + if agp_line.obj != self._current_obj: + + # Check if we have already seen this object before + if agp_line.obj in self._seen_objs: + raise AGPError(self.fname, agp_line.line_number, "object identifier out of order") + + # Add the new object to our master list + agp_obj = AGPObject(self.fname, agp_line) + self._objects.append(agp_obj) + + # Initialize all the info for this new object + self._current_obj = agp_obj.obj + self._seen_objs.add(agp_obj.obj) + + else: + self._objects[-1].add_line(agp_line) + + @property + def agp_version(self): + return self._agp_version + + @property + def fname(self): + return self._fname + + @property + def num_lines(self): + """ Calculate the number of lines in the current state of the AGP file. """ + return sum([len(self._comment_lines)] + [obj.num_lines for obj in self._objects]) + + @property + def num_objs(self): + return len(self._objects) + + def add_pragma(self): + pragma = "## agp-version {}".format(self.agp_version) + if self._comment_lines: + new_comment_lines = [pragma] + for i in self._comment_lines: + if i != pragma: + new_comment_lines.append(i) + self._comment_lines = new_comment_lines + else: + self._comment_lines.append(pragma) + + def iterate_objs(self): + """ Iterate over the objects of the AGP file. """ + for obj in self._objects: + yield obj + + def iterate_lines(self): + """ Iterate over the non-comment lines of AGP file. """ + for obj in self.iterate_objs(): + for j in obj.iterate_lines(): + yield j + + def add_comment(self, c): + if not isinstance(c, str): + raise TypeError("Comment must be a string") + + if not c.startswith("#"): + raise ValueError("Comment must start with a '#' character") + + if c not in self._comment_lines: + self._comment_lines.append(c) + + def add_seq_line(self, obj, obj_beg, obj_end, pid, comp_type, comp, comp_beg, comp_end, orientation): + """ + # TODO fill this out + :param obj: + :param obj_beg: + :param obj_end: + :param pid: + :param comp_type: + :param comp: + :param comp_beg: + :param comp_end: + :param orientation: + """ + line_number = self.num_lines + 1 + agp_line = AGPSeqLine(self.fname, line_number, obj, obj_beg, obj_end, pid, comp_type, comp, comp_beg, comp_end, orientation) + self._add_line(agp_line) + + def add_gap_line(self, obj, obj_beg, obj_end, pid, comp_type, gap_len, gap_type, linkage, linkage_evidence): + """ + # TODO fill this out + :param obj: + :param obj_beg: + :param obj_end: + :param pid: + :param comp_type: + :param gap_len: + :param gap_type: + :param linkage: + :param linkage_evidence: + """ + line_number = self.num_lines + 1 + agp_line = AGPGapLine(self.fname, line_number, obj, obj_beg, obj_end, pid, comp_type, gap_len, gap_type, linkage, linkage_evidence) + self._add_line(agp_line) + + def pop_agp_line(self): + """ Remove the last AGP line and update state info accordingly. """ + if not self._objects: + return + + if self._objects[-1].num_lines == 1: + self._seen_objs.remove(self._objects[-1].obj) + self._objects = self._objects[:-1] + + else: + self._objects[-1].pop_line() + + def write(self): + """ Write the agp contents to a file. """ + with open(self.fname, "w") as f: + if self._comment_lines: + f.write("\n".join(self._comment_lines) + "\n") + if self._objects: + f.write("\n".join([str(obj) for obj in self._objects]) + "\n") + + +class AGPObject: + + """ + This (python) object represents an AGP object. Objects will consist of AGP lines, and have to adhere to + certain rules. By organizing AGP lines into the objects that they comprise, we can easily calculate stats + about the assembly (the collection of objects). + """ + + def __init__(self, agp_fname, in_agp_line): + if not isinstance(in_agp_line, AGPLine): + raise TypeError("in_agp_line must be an instance of 'AGPLine'") + + # The object is defined by the object identifier and a list of AGP lines + self.fname = agp_fname + self._obj = in_agp_line.obj + self._agp_lines = [] + + # Store info enabling us to keep track of the state of the object + self.previous_pid = 0 + self.obj_intervals = [] # Stores intervals as 0-indexed + + # Perform checks to ensure the object is properly initialized + if not in_agp_line.obj_beg == 1: + raise AGPError(self.fname, in_agp_line.line_number, "the first object coordinates should start with '1'") + + if not in_agp_line.pid == 1: + raise AGPError(self.fname, in_agp_line.line_number, "all objects should start with a 'part_number' of '1'") + + # If we have passed the initialization tests, add this line like any other + self.add_line(in_agp_line) + + def __str__(self): + return "\n".join([str(i) for i in self._agp_lines]) + + def __repr__(self): + return "AGP Object: {}".format(self.obj) + + def __iter__(self): + for line in self._agp_lines: + yield dict(line) + + @property + def obj(self): + return self._obj + + @property + def obj_len(self): + return int(self.obj_intervals[-1][1]) + + @property + def num_lines(self): + return len(self._agp_lines) + + def add_line(self, agp_line): + # Perform validity checks if this is a new object + if agp_line.obj != self.obj: + raise AGPError(self.fname, agp_line, "cannot add line from object {} to object {}".format(agp_line.obj, self.obj)) + + # Check that our PID is sequential + if agp_line.pid - self.previous_pid != 1: + raise AGPError(self.fname, agp_line.line_number, "non-sequential part_numbers") + + # Check that the object intervals are sequential + if self.obj_intervals: + if self.obj_intervals[-1][1] != agp_line.obj_beg - 1: + raise AGPError(self.fname, agp_line.line_number, "some positions in %s are not accounted for or overlapping" % agp_line.obj) + + self.previous_pid = agp_line.pid + self.obj_intervals.append((agp_line.obj_beg - 1, agp_line.obj_end)) + self._agp_lines.append(agp_line) + + def iterate_lines(self): + for i in self._agp_lines: + yield i + + +class AGPLine: + + """ + An abstract base class representing a single AGP file line. Inheriting subclasses should + override or implement new methods to check the validity of a single AFP line. Validity + checks that involve multiple lines should not be considered. + """ + + __metaclass__ = abc.ABCMeta + allowed_comp_types = set() + + def __init__(self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type): + self.is_gap = None + + # File info + self.fname = fname + self.line_number = line_number + # Object info + self.obj = obj + self.obj_beg = obj_beg + self.obj_end = obj_end + self.pid = pid + self.comp_type = comp_type + + self._validate_numerics() + self._validate_strings() + self._validate_obj_coords() + self._validate_component_type() + self._validate_line() + + @abc.abstractmethod + def __str__(self): + """ Return the tab delimited AGP line""" + pass + + @abc.abstractmethod + def __iter__(self): + """ Return the AGP line's iterator""" + pass + + @abc.abstractmethod + def _validate_numerics(self): + """ Ensure all numeric fields and positive integers. """ + pass + + @abc.abstractmethod + def _validate_strings(self): + """ Ensure all text fields are strings. """ + pass + + def _validate_obj_coords(self): + if self.obj_beg > self.obj_end: + raise AGPError(self.fname, self.line_number, "object_beg (%d) must be <= object_end (%d)" % (self.obj_beg, self.obj_end)) + + def _validate_component_type(self): + if self.comp_type not in self.allowed_comp_types: + raise AGPError(self.fname, self.line_number, "invalid component type: %s" % self.comp_type) + + @abc.abstractmethod + def _validate_line(self): + """ Final remaining validations specific to the gap or sequence AGP lines. """ + pass + + +class AGPSeqLine(AGPLine): + + """ + A subclass of AGPLine specifically for AGP lines that represent sequences. + """ + + allowed_comp_types = {"A", "D", "F", "G", "O", "P", "W"} + allowed_orientations = {"+", "-", "?", "0", "na"} + + def __init__(self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type, comp, comp_beg, comp_end, orientation): + self.comp = comp + self.comp_beg = comp_beg + self.comp_end = comp_end + self.orientation = orientation + + # Set the object attributes and perform superclass-defined validations + super(AGPSeqLine, self).__init__(fname, line_number, obj, obj_beg, obj_end, pid, comp_type) + + self.is_gap = False + self.seqdict = dict(obj=str(self.obj), + obj_beg=int(self.obj_beg), + obj_end=int(self.obj_end), + pid=int(self.pid), + comp_type=str(self.comp_type), + comp=str(self.comp), + comp_beg=int(self.comp_beg), + comp_end=int(self.comp_end), + orientation=str(self.orientation)) + + def __str__(self): + return "\t".join([ + self.obj, + str(self.obj_beg), + str(self.obj_end), + str(self.pid), + self.comp_type, + self.comp, + str(self.comp_beg), + str(self.comp_end), + self.orientation + ]) + + def __iter__(self): + for key in self.seqdict: + yield str(key), self.seqdict[key] + + def _validate_numerics(self): + # Convert all numeric types to integers + try: + self.line_number = int(self.line_number) + self.obj_beg = int(self.obj_beg) + self.obj_end = int(self.obj_end) + self.pid = int(self.pid) + self.comp_beg = int(self.comp_beg) + self.comp_end = int(self.comp_end) + except TypeError: + raise AGPError(self.fname, self.line_number, "encountered an invalid non-integer numeric AGP field") + + # Ensure that all numeric values are positive + if not all([ + self.obj_beg > 0, + self.obj_end > 0, + self.pid > 0, + self.comp_beg > 0, + self.comp_end > 0 + ]): + raise AGPError(self.fname, self.line_number, "encountered an invalid zero or negative numeric AGP field.") + + # Check the coordinates + if self.comp_beg > self.comp_end: + raise AGPError(self.fname, self.line_number, "component_beg (%d) must be <= component_end (%d)" % (self.comp_beg, self.comp_end)) + + if self.obj_end - (self.obj_beg-1) != self.comp_end - (self.comp_beg-1): + raise AGPError(self.fname, self.line_number, "object coordinates (%d, %d) and component coordinates (%d, %d) do not have the same length" % (self.obj_beg, self.obj_end, self.comp_beg, self.comp_end)) + + def _validate_strings(self): + try: + self.obj = str(self.obj) + self.comp_type = str(self.comp_type) + self.comp = str(self.comp) + self.orientation = str(self.orientation) + except TypeError: + raise AGPError(self.fname, self.line_number, "encountered an invalid type for an AGP text field") + + def _validate_line(self): + if self.orientation not in AGPSeqLine.allowed_orientations: + raise AGPError(self.fname, self.line_number, "invalid orientation: %s" % self.orientation) + + +class AGPGapLine(AGPLine): + + """ + A subclass of AGPLine specifically for AGP lines that represent sequence gaps. + """ + + allowed_comp_types = {"N", "U"} + allowed_linkage_types = {"yes", "no"} + allowed_gap_types = { + "scaffold", "contig", "centromere", "short_arm", "heterochromatin", "telomere", "repeat", "contamination" + } + allowed_evidence_types = { + "na", "paired-ends", "align_genus", "align_xgenus", + "align_trnscpt", "within_clone", "clone_contig", "map", + "pcr", "proximity_ligation", "strobe", "unspecified" + } + + def __init__(self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type, gap_len, gap_type, linkage, linkage_evidence): + self.gap_len = gap_len + self.gap_type = gap_type + self.linkage = linkage + self.linkage_evidence = linkage_evidence + + # Set the object attributes and perform superclass-defined validations + super(AGPGapLine, self).__init__(fname, line_number, obj, obj_beg, obj_end, pid, comp_type) + + self.is_gap = True + self.gapdict = dict(obj=str(self.obj), + obj_beg=int(self.obj_beg), + obj_end=int(self.obj_end), + pid=int(self.pid), + comp_type=str(self.comp_type), + gap_len=int(self.gap_len), + gap_type=str(self.gap_type), + linkage=str(self.linkage), + linkage_evidence=str(self.linkage_evidence)) + + + def __str__(self): + return "\t".join([ + self.obj, + str(self.obj_beg), + str(self.obj_end), + str(self.pid), + self.comp_type, + str(self.gap_len), + self.gap_type, + self.linkage, + self.linkage_evidence + ]) + + def __iter__(self): + for key in self.gapdict: + yield str(key), self.gapdict[key] + + def _validate_numerics(self): + # Convert all numeric types to integers + try: + self.line_number = int(self.line_number) + self.obj_beg = int(self.obj_beg) + self.obj_end = int(self.obj_end) + self.pid = int(self.pid) + self.gap_len = int(self.gap_len) + except TypeError: + raise AGPError(self.fname, self.line_number, "encountered an invalid non-integer numeric AGP field") + + # Ensure that all numeric values are positive + if not all([ + self.obj_beg > 0, + self.obj_end > 0, + self.pid > 0, + self.gap_len > 0 + ]): + raise AGPError(self.fname, self.line_number, "encountered an invalid negative numeric AGP field") + + # Make sure the coordinates match + if self.obj_end - (self.obj_beg-1) != self.gap_len: + raise AGPError(self.fname, self.line_number, "object coordinates (%d, %d) and gap length (%d) are not the same length" % (self.obj_beg, self.obj_end, self.gap_len)) + + def _validate_strings(self): + try: + self.obj = str(self.obj) + self.comp_type = str(self.comp_type) + self.gap_type = str(self.gap_type) + self.linkage = str(self.linkage) + self.linkage_evidence = str(self.linkage_evidence) + except TypeError: + raise AGPError(self.fname, self.line_number, "encountered an invalid type for an AGP text field") + + def _validate_line(self): + """ Validation specific to AGP gap lines. """ + if self.comp_type == "U" and self.gap_len != 100: + raise AGPError(self.fname, self.line_number, "invalid gap length for component type 'U': %d (should be 100)" % self.gap_len) + + if self.gap_type not in AGPGapLine.allowed_gap_types: + raise AGPError(self.fname, self.line_number, "invalid gap type: %s" % self.gap_type) + + if self.linkage not in AGPGapLine.allowed_linkage_types: + raise AGPError(self.fname, self.line_number, "invalid linkage field: %s" % self.linkage) + + all_evidence = self.linkage_evidence.split(";") + for e in all_evidence: + if e not in AGPGapLine.allowed_evidence_types: + raise AGPError(self.fname, self.line_number, "invalid linkage evidence: %s" % e) + + if self.linkage == "no": + if self.gap_type == "scaffold": + raise AGPError(self.fname, self.line_number, "invalid 'scaffold' gap without linkage evidence") + + if self.linkage_evidence != "na": + raise AGPError(self.fname, self.line_number, "linkage evidence must be 'na' when not asserting linkage. Got {}".format(self.linkage_evidence)) + else: + if "na" in all_evidence: + raise AGPError(self.fname, self.line_number, "'na' is invalid linkage evidence when asserting linkage") diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 360e60afc2a..cb6eb4ece88 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -460,12 +460,17 @@ class Taxonomy(Tabular): return self.make_html_table(dataset, column_names=self.column_names) +@dataproviders.decorators.has_dataproviders @build_sniff_from_prefix class GoldenPath(Tabular): """Class describing NCBI's Golden Path assembly format""" edam_format = 'format_3693' file_ext = 'agp' + @dataproviders.decorators.dataprovider_factory('dict', dataproviders.column.GoldenPathDataProvider.settings) + def dict_dataprovider(self, dataset, **settings): + return super().dict_dataprovider(dataset, **settings) + def sniff_prefix(self, file_prefix): """ Checks for and does cursory validation on data that looks like AGP diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 3393db81073..6e5c682491c 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -17,6 +17,8 @@ from bx.bbi.bigbed_file import BigBedFile from bx.bbi.bigwig_file import BigWigFile from bx.interval_index_file import Indexes +from galaxy.datatypes.dataproviders import column +from galaxy.datatypes.goldenpath import AGPFile from galaxy.datatypes.interval import Bed, Gff, Gtf from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFFeature, GFFInterval, GFFReaderWrapper, parse_gff_attributes from galaxy.visualization.data_providers.basic import BaseDataProvider @@ -312,6 +314,26 @@ class FilterableMixin: return filters +class GoldenPathDataProvider(column.GoldenPathDataProvider): + ''' + Class that defines a data provider for AGP datasets. + ''' + dataset_type: dict + + def __init__(self, original_dataset, **kwargs): + filename = original_dataset.dataset.get_file_name() + self.goldenpath = AGPFile(filename) + self.columns = [str(gp) for gp in self.goldenpath] + super().__init__(filename, columns=self.columns, **kwargs) + + def __iter__(self): + for gp in self.goldenpath: + yield gp + + def get_data(self, columns=None): + return dict(self.goldenpath) + + class TabixDataProvider(GenomeDataProvider, FilterableMixin): dataset_type = 'tabix' diff --git a/lib/galaxy/visualization/data_providers/registry.py b/lib/galaxy/visualization/data_providers/registry.py index ad573b80608..51a87c7ec6d 100644 --- a/lib/galaxy/visualization/data_providers/registry.py +++ b/lib/galaxy/visualization/data_providers/registry.py @@ -7,7 +7,7 @@ from galaxy.datatypes.interval import ( Gtf, Interval ) -from galaxy.datatypes.tabular import Tabular, Vcf +from galaxy.datatypes.tabular import Tabular, Vcf, GoldenPath from galaxy.datatypes.xml import Phyloxml from galaxy.model import NoConverterException from galaxy.visualization.data_providers import genome @@ -35,6 +35,7 @@ class DataProviderRegistry: "default": genome.TabixDataProvider }, "interval_index": genome.IntervalIndexDataProvider, + "agp": genome.GoldenPathDataProvider, "bai": genome.BamDataProvider, "bam": genome.SamDataProvider, "bigwig": genome.BigWigDataProvider, @@ -58,6 +59,13 @@ class DataProviderRegistry: data_provider_class = genome.RawBedDataProvider elif isinstance(original_dataset.datatype, Vcf): data_provider_class = genome.RawVcfDataProvider + # Quick reminder, to save the next developer a few minutes of + # debugging time, that any datatype class that is a subclass of + # another needs to be checked before the parent in this conditional. + elif isinstance(original_dataset.datatype, GoldenPath): + data_provider_class = genome.GoldenPathDataProvider + # Otherwise GoldenPath, being a subclass of Tabular, would match + # this clause and we would return the wrong data provider class. elif isinstance(original_dataset.datatype, Tabular): data_provider_class = ColumnDataProvider elif isinstance(original_dataset.datatype, (Nexus, Newick, Phyloxml)): From 8761f8f6e5e344656fd7c0aa458cd2d9b281aa31 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Fri, 8 Oct 2021 14:54:09 -0400 Subject: [PATCH 024/781] Fix linting errors. --- lib/galaxy/datatypes/dataproviders/column.py | 1 - lib/galaxy/datatypes/goldenpath.py | 10 ++++------ lib/galaxy/visualization/data_providers/registry.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index d03c1f30809..c3acd99af92 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -7,7 +7,6 @@ import re from urllib.parse import unquote_plus from galaxy.datatypes.goldenpath import AGPFile - from . import line _TODO = """ diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py index c735239186d..9eb385b3e4a 100755 --- a/lib/galaxy/datatypes/goldenpath.py +++ b/lib/galaxy/datatypes/goldenpath.py @@ -22,8 +22,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -import os import abc +import os class AGPError(Exception): @@ -331,7 +331,7 @@ class AGPObject: yield i -class AGPLine: +class AGPLine(object, metaclass=abc.ABCMeta): """ An abstract base class representing a single AGP file line. Inheriting subclasses should @@ -339,7 +339,6 @@ class AGPLine: checks that involve multiple lines should not be considered. """ - __metaclass__ = abc.ABCMeta allowed_comp_types = set() def __init__(self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type): @@ -467,7 +466,7 @@ class AGPSeqLine(AGPLine): if self.comp_beg > self.comp_end: raise AGPError(self.fname, self.line_number, "component_beg (%d) must be <= component_end (%d)" % (self.comp_beg, self.comp_end)) - if self.obj_end - (self.obj_beg-1) != self.comp_end - (self.comp_beg-1): + if self.obj_end - (self.obj_beg - 1) != self.comp_end - (self.comp_beg - 1): raise AGPError(self.fname, self.line_number, "object coordinates (%d, %d) and component coordinates (%d, %d) do not have the same length" % (self.obj_beg, self.obj_end, self.comp_beg, self.comp_end)) def _validate_strings(self): @@ -521,7 +520,6 @@ class AGPGapLine(AGPLine): linkage=str(self.linkage), linkage_evidence=str(self.linkage_evidence)) - def __str__(self): return "\t".join([ self.obj, @@ -560,7 +558,7 @@ class AGPGapLine(AGPLine): raise AGPError(self.fname, self.line_number, "encountered an invalid negative numeric AGP field") # Make sure the coordinates match - if self.obj_end - (self.obj_beg-1) != self.gap_len: + if self.obj_end - (self.obj_beg - 1) != self.gap_len: raise AGPError(self.fname, self.line_number, "object coordinates (%d, %d) and gap length (%d) are not the same length" % (self.obj_beg, self.obj_end, self.gap_len)) def _validate_strings(self): diff --git a/lib/galaxy/visualization/data_providers/registry.py b/lib/galaxy/visualization/data_providers/registry.py index 51a87c7ec6d..760acad21e3 100644 --- a/lib/galaxy/visualization/data_providers/registry.py +++ b/lib/galaxy/visualization/data_providers/registry.py @@ -7,7 +7,7 @@ from galaxy.datatypes.interval import ( Gtf, Interval ) -from galaxy.datatypes.tabular import Tabular, Vcf, GoldenPath +from galaxy.datatypes.tabular import GoldenPath, Tabular, Vcf from galaxy.datatypes.xml import Phyloxml from galaxy.model import NoConverterException from galaxy.visualization.data_providers import genome From 25c4ad96d362e9799e9bbafbea48a05bea210836 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 12 Oct 2021 09:24:04 -0400 Subject: [PATCH 025/781] Fix mypy errors. --- lib/galaxy/datatypes/dataproviders/column.py | 3 ++- lib/galaxy/datatypes/goldenpath.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index c3acd99af92..bd172e46a97 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -5,6 +5,7 @@ is further subdivided into multiple data (e.g. columns from a line). import logging import re from urllib.parse import unquote_plus +from typing import Dict from galaxy.datatypes.goldenpath import AGPFile from . import line @@ -360,7 +361,7 @@ class GoldenPathDataProvider(DictDataProvider): ''' Class that defines a data provider for AGP datasets. ''' - settings = {} + settings = {} # type: Dict[str, str] def __init__(self, source, **kwargs): self.goldenpath = AGPFile(source) diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py index 9eb385b3e4a..abbcc23dde7 100755 --- a/lib/galaxy/datatypes/goldenpath.py +++ b/lib/galaxy/datatypes/goldenpath.py @@ -25,6 +25,8 @@ SOFTWARE. import abc import os +from typing import Set + class AGPError(Exception): """ Exception raised for AGP related errors. """ @@ -339,7 +341,7 @@ class AGPLine(object, metaclass=abc.ABCMeta): checks that involve multiple lines should not be considered. """ - allowed_comp_types = set() + allowed_comp_types = set() # type: Set[str] def __init__(self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type): self.is_gap = None From eceddfe454dad2dcf58c63079c56fe4dfeafb39a Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 12 Oct 2021 10:22:14 -0400 Subject: [PATCH 026/781] =?UTF-8?q?Implement=20suggestions=20from=20review?= =?UTF-8?q?=20by=20David=20L=C3=B3pez.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/galaxy/datatypes/dataproviders/column.py | 2 +- lib/galaxy/datatypes/goldenpath.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index bd172e46a97..f419a8fe95c 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -361,7 +361,7 @@ class GoldenPathDataProvider(DictDataProvider): ''' Class that defines a data provider for AGP datasets. ''' - settings = {} # type: Dict[str, str] + settings: Dict[str, str] = {} def __init__(self, source, **kwargs): self.goldenpath = AGPFile(source) diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py index abbcc23dde7..6b0872265fb 100755 --- a/lib/galaxy/datatypes/goldenpath.py +++ b/lib/galaxy/datatypes/goldenpath.py @@ -25,7 +25,7 @@ SOFTWARE. import abc import os -from typing import Set +from typing import Set, Union class AGPError(Exception): @@ -126,6 +126,7 @@ class AGPFile: if not all(fields): raise AGPError(self.fname, line_number, "detected an empty field") + agp_line: Union[AGPGapLine, AGPSeqLine] # Instantiate all the AGPLine objects. These will do line-specific validations. if fields[4] == "N" or fields[4] == "U": agp_line = AGPGapLine(self.fname, line_number, *fields) @@ -341,7 +342,7 @@ class AGPLine(object, metaclass=abc.ABCMeta): checks that involve multiple lines should not be considered. """ - allowed_comp_types = set() # type: Set[str] + allowed_comp_types: Set[str] = set() def __init__(self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type): self.is_gap = None From 09a02981c40d88611e230d586664671cd4972e6a Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 12 Oct 2021 10:34:32 -0400 Subject: [PATCH 027/781] Use f-strings for formatting. --- lib/galaxy/datatypes/goldenpath.py | 35 ++++++++++++++---------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py index 6b0872265fb..a8faa155e1f 100755 --- a/lib/galaxy/datatypes/goldenpath.py +++ b/lib/galaxy/datatypes/goldenpath.py @@ -36,10 +36,7 @@ class AGPError(Exception): self.line_number = line_number self.message = message - self.report = "" - self.report += "\n\nFILE : {}\n".format(self.fname) - self.report += "LINE : {}\n".format(self.line_number) - self.report += "ERROR: {}\n".format(self.message) + self.report = f"\n\nFILE: {self.fname}\nLINE: {self.line_number}\nERROR: {self.message}" super().__init__(self.report) def __repr__(self): @@ -172,7 +169,7 @@ class AGPFile: return len(self._objects) def add_pragma(self): - pragma = "## agp-version {}".format(self.agp_version) + pragma = f"## agp-version {self.agp_version}" if self._comment_lines: new_comment_lines = [pragma] for i in self._comment_lines: @@ -293,7 +290,7 @@ class AGPObject: return "\n".join([str(i) for i in self._agp_lines]) def __repr__(self): - return "AGP Object: {}".format(self.obj) + return f"AGP Object: {self.obj}" def __iter__(self): for line in self._agp_lines: @@ -314,7 +311,7 @@ class AGPObject: def add_line(self, agp_line): # Perform validity checks if this is a new object if agp_line.obj != self.obj: - raise AGPError(self.fname, agp_line, "cannot add line from object {} to object {}".format(agp_line.obj, self.obj)) + raise AGPError(self.fname, agp_line, f"cannot add line from object {agp_line.obj} to object {self.obj}") # Check that our PID is sequential if agp_line.pid - self.previous_pid != 1: @@ -323,7 +320,7 @@ class AGPObject: # Check that the object intervals are sequential if self.obj_intervals: if self.obj_intervals[-1][1] != agp_line.obj_beg - 1: - raise AGPError(self.fname, agp_line.line_number, "some positions in %s are not accounted for or overlapping" % agp_line.obj) + raise AGPError(self.fname, agp_line.line_number, f"some positions in {agp_line.obj} are not accounted for or overlapping") self.previous_pid = agp_line.pid self.obj_intervals.append((agp_line.obj_beg - 1, agp_line.obj_end)) @@ -385,11 +382,11 @@ class AGPLine(object, metaclass=abc.ABCMeta): def _validate_obj_coords(self): if self.obj_beg > self.obj_end: - raise AGPError(self.fname, self.line_number, "object_beg (%d) must be <= object_end (%d)" % (self.obj_beg, self.obj_end)) + raise AGPError(self.fname, self.line_number, f"object_beg ({self.obj_beg}) must be <= object_end ({self.obj_end})") def _validate_component_type(self): if self.comp_type not in self.allowed_comp_types: - raise AGPError(self.fname, self.line_number, "invalid component type: %s" % self.comp_type) + raise AGPError(self.fname, self.line_number, f"invalid component type: {self.comp_type}") @abc.abstractmethod def _validate_line(self): @@ -467,10 +464,10 @@ class AGPSeqLine(AGPLine): # Check the coordinates if self.comp_beg > self.comp_end: - raise AGPError(self.fname, self.line_number, "component_beg (%d) must be <= component_end (%d)" % (self.comp_beg, self.comp_end)) + raise AGPError(self.fname, self.line_number, f"component_beg ({self.comp_beg}) must be <= component_end ({self.comp_end})") if self.obj_end - (self.obj_beg - 1) != self.comp_end - (self.comp_beg - 1): - raise AGPError(self.fname, self.line_number, "object coordinates (%d, %d) and component coordinates (%d, %d) do not have the same length" % (self.obj_beg, self.obj_end, self.comp_beg, self.comp_end)) + raise AGPError(self.fname, self.line_number, f"object coordinates ({self.obj_beg}, {self.obj_end}) and component coordinates ({self.comp_beg}, {self.comp_end}) do not have the same length") def _validate_strings(self): try: @@ -483,7 +480,7 @@ class AGPSeqLine(AGPLine): def _validate_line(self): if self.orientation not in AGPSeqLine.allowed_orientations: - raise AGPError(self.fname, self.line_number, "invalid orientation: %s" % self.orientation) + raise AGPError(self.fname, self.line_number, f"invalid orientation: {self.orientation}") class AGPGapLine(AGPLine): @@ -562,7 +559,7 @@ class AGPGapLine(AGPLine): # Make sure the coordinates match if self.obj_end - (self.obj_beg - 1) != self.gap_len: - raise AGPError(self.fname, self.line_number, "object coordinates (%d, %d) and gap length (%d) are not the same length" % (self.obj_beg, self.obj_end, self.gap_len)) + raise AGPError(self.fname, self.line_number, f"object coordinates ({self.obj_beg}, {self.obj_end}) and gap length ({self.gap_len}) are not the same length") def _validate_strings(self): try: @@ -577,25 +574,25 @@ class AGPGapLine(AGPLine): def _validate_line(self): """ Validation specific to AGP gap lines. """ if self.comp_type == "U" and self.gap_len != 100: - raise AGPError(self.fname, self.line_number, "invalid gap length for component type 'U': %d (should be 100)" % self.gap_len) + raise AGPError(self.fname, self.line_number, f"invalid gap length for component type 'U': {self.gap_len} (should be 100)") if self.gap_type not in AGPGapLine.allowed_gap_types: - raise AGPError(self.fname, self.line_number, "invalid gap type: %s" % self.gap_type) + raise AGPError(self.fname, self.line_number, f"invalid gap type: {self.gap_type}") if self.linkage not in AGPGapLine.allowed_linkage_types: - raise AGPError(self.fname, self.line_number, "invalid linkage field: %s" % self.linkage) + raise AGPError(self.fname, self.line_number, f"invalid linkage field: {self.linkage}") all_evidence = self.linkage_evidence.split(";") for e in all_evidence: if e not in AGPGapLine.allowed_evidence_types: - raise AGPError(self.fname, self.line_number, "invalid linkage evidence: %s" % e) + raise AGPError(self.fname, self.line_number, f"invalid linkage evidence: {e}") if self.linkage == "no": if self.gap_type == "scaffold": raise AGPError(self.fname, self.line_number, "invalid 'scaffold' gap without linkage evidence") if self.linkage_evidence != "na": - raise AGPError(self.fname, self.line_number, "linkage evidence must be 'na' when not asserting linkage. Got {}".format(self.linkage_evidence)) + raise AGPError(self.fname, self.line_number, f"linkage evidence must be 'na' when not asserting linkage. Got {self.linkage_evidence}") else: if "na" in all_evidence: raise AGPError(self.fname, self.line_number, "'na' is invalid linkage evidence when asserting linkage") From 3bcba80c3beffe90d9b2d665d80d95498053c50a Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 12 Oct 2021 10:39:47 -0400 Subject: [PATCH 028/781] Flake8 fixes. --- lib/galaxy/datatypes/dataproviders/column.py | 2 +- lib/galaxy/datatypes/goldenpath.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index f419a8fe95c..9313d810996 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -4,8 +4,8 @@ is further subdivided into multiple data (e.g. columns from a line). """ import logging import re -from urllib.parse import unquote_plus from typing import Dict +from urllib.parse import unquote_plus from galaxy.datatypes.goldenpath import AGPFile from . import line diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py index a8faa155e1f..c8f47f6e0ff 100755 --- a/lib/galaxy/datatypes/goldenpath.py +++ b/lib/galaxy/datatypes/goldenpath.py @@ -24,7 +24,6 @@ SOFTWARE. import abc import os - from typing import Set, Union From 90547b39c537e03b229f4977a3843a58f786b1d5 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 9 Nov 2021 20:23:34 -0500 Subject: [PATCH 029/781] Add 'after' endpoint for history_contents --- .../webapps/galaxy/api/history_contents.py | 16 +++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 4 +++ .../galaxy/services/history_contents.py | 34 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 23ec3049881..c03b01ea011 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -518,6 +518,22 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, trans, history_id, serialization_params, filter_params, hid, limit, since, ) + @expose_api_raw_anonymous + def contents_after(self, trans, history_id, hid, limit, **kwd): + """ + Return {limit} history items with hid > {hid}. + + GET /api/histories/{history_id}/contents/after/{hid}/{limit} + """ + serialization_params = parse_serialization_params(default_view='betawebclient', **kwd) + filter_params = self._parse_rest_params(kwd) + hid = int(hid) + limit = int(limit) + + return self.service.contents_after( + trans, history_id, serialization_params, filter_params, hid, limit + ) + # Parsing query string according to REST standards. def _parse_rest_params(self, qdict: Dict[str, Any]) -> HistoryContentsFilterList: DEFAULT_OP = 'eq' diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 97d62d70aca..8e073df6a2e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -317,6 +317,10 @@ def populate_api_routes(webapp, app): action="contents_near", controller='history_contents', conditions=dict(method=["GET"])) + webapp.mapper.connect("/api/histories/{history_id}/contents/after/{hid}/{limit}", + action="contents_after", + controller='history_contents', + conditions=dict(method=["GET"])) webapp.mapper.resource('user', 'users', controller='group_users', diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index b2cbe6fea9a..bb184e28300 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -756,6 +756,40 @@ class HistoriesContentsService(ServiceBase): return json.dumps(contents) + def contents_after( + self, trans, + history_id: EncodedDatabaseIdField, + serialization_params: SerializationParams, + filter_params: HistoryContentsFilterList, + hid: int, + limit: int, + since: Optional[datetime.datetime] = None, + ): + """ + Return {limit} history items with hid > {hid}. + """ + history: History = self.history_manager.get_accessible( + self.decode_id(history_id), trans.user, current_history=trans.history + ) + up_params = filter_params + self._hid_greater_than(hid) + up_order = 'hid-asc' + contents_up, up_count = self._seek(history, up_params, up_order, limit, serialization_params) + + min_hid, max_hid = self._get_filtered_extrema(history, filter_params) + + contents = self._expand_contents(trans, contents_up, serialization_params) + contents.reverse() + + # Put stats in http headers + trans.response.headers['matches_up'] = len(contents_up) + trans.response.headers['total_matches_up'] = up_count + trans.response.headers['max_hid'] = max_hid + trans.response.headers['min_hid'] = min_hid + trans.response.headers['history_size'] = str(history.disk_size) + trans.response.headers['history_empty'] = json.dumps(history.empty) # convert to proper bool + + return json.dumps(contents) + def _hid_greater_than(self, hid: int) -> HistoryContentsFilterList: return [["hid", "gt", hid]] From 9ea2837aea404069ed9049ca78569cd8051ab994 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Wed, 10 Nov 2021 17:47:25 +1100 Subject: [PATCH 030/781] fix parsing of yml job conf file in galaxy.dependencies --- lib/galaxy/dependencies/__init__.py | 12 +++++------- test/unit/app/dependencies/test_deps.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 52e9ec331e3..212a2c1ad6c 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -42,15 +42,13 @@ class ConditionalDependencies: def parse_configs(self): def load_job_config_dict(job_conf_dict): - for runner in job_conf_dict.get("runners"): + runners = job_conf_dict.get("runners", []) + for runner_id in runners: + runner = runners[runner_id] if "load" in runner: self.job_runners.append(runner.get("load")) if "rules_module" in runner: - self.job_rule_modules.append(plugin.text) - if "params" in runner: - runner_params = runner["params"] - if "rules_module" in runner_params: - self.job_rule_modules.append(plugin.text) + self.job_rule_modules.append(runner.get("rules_module")) if "job_config" in self.config: load_job_config_dict(self.config.get("job_config")) @@ -75,7 +73,7 @@ class ConditionalDependencies: pass else: try: - with open("job_conf_path") as f: + with open(job_conf_path) as f: job_conf_dict = yaml.safe_load(f) load_job_config_dict(job_conf_dict) except OSError: diff --git a/test/unit/app/dependencies/test_deps.py b/test/unit/app/dependencies/test_deps.py index e8ac9fcefbb..c97f81a18e0 100644 --- a/test/unit/app/dependencies/test_deps.py +++ b/test/unit/app/dependencies/test_deps.py @@ -24,6 +24,11 @@ FILES_SOURCES_DROPBOX = """ - type: webdav - type: dropbox """ +JOB_CONF_YAML = """ +runners: + runner1: + load: job_runner_A +""" def test_default_objectstore(): @@ -80,6 +85,16 @@ def test_fs_configured(): assert cds.check_fs_webdavfs() +def test_yaml_jobconf_runners(): + with _config_context() as cc: + job_conf_file = cc.write_config("job_conf.yml", JOB_CONF_YAML) + config = { + "job_config_file": job_conf_file, + } + cds = cc.get_cond_deps(config=config) + assert 'job_runner_A' in cds.job_runners + + @contextmanager def _config_context(): config_dir = mkdtemp() From fc04ed5bef1224c146dee790a7a564cb37cda27f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 9 Nov 2021 23:33:17 +0100 Subject: [PATCH 031/781] Remove currently unused data provider, validate file during metadata process --- .../config/sample/datatypes_conf.xml.sample | 4 +- lib/galaxy/datatypes/dataproviders/column.py | 18 -- lib/galaxy/datatypes/goldenpath.py | 186 +++++++----------- lib/galaxy/datatypes/tabular.py | 54 ----- .../visualization/data_providers/genome.py | 22 --- .../visualization/data_providers/registry.py | 13 +- 6 files changed, 73 insertions(+), 224 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 00d36c8fbe2..c58a40f84c9 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -4,7 +4,7 @@ - + @@ -892,7 +892,7 @@ - + diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index 9313d810996..6a94597bcf4 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -4,10 +4,8 @@ is further subdivided into multiple data (e.g. columns from a line). """ import logging import re -from typing import Dict from urllib.parse import unquote_plus -from galaxy.datatypes.goldenpath import AGPFile from . import line _TODO = """ @@ -355,19 +353,3 @@ class DictDataProvider(ColumnarDataProvider): for column_values in parent_gen: map = dict(zip(self.column_names, column_values)) yield map - - -class GoldenPathDataProvider(DictDataProvider): - ''' - Class that defines a data provider for AGP datasets. - ''' - settings: Dict[str, str] = {} - - def __init__(self, source, **kwargs): - self.goldenpath = AGPFile(source) - self.column_names = [str(gp) for gp in self.goldenpath] - super().__init__(source, **kwargs) - - def __iter__(self): - for gp in self.goldenpath: - yield gp diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py index c8f47f6e0ff..d59f247d070 100755 --- a/lib/galaxy/datatypes/goldenpath.py +++ b/lib/galaxy/datatypes/goldenpath.py @@ -1,34 +1,73 @@ -""" -MIT License - -Copyright (c) 2021 Michael Alonge - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - import abc import os -from typing import Set, Union +from typing import ( + Set, + Union, +) + +from galaxy.datatypes.sniff import ( + build_sniff_from_prefix, + iter_headers, +) +from .tabular import Tabular + + +@build_sniff_from_prefix +class GoldenPath(Tabular): + """Class describing NCBI's Golden Path assembly format""" + edam_format = 'format_3693' + file_ext = 'agp' + + def set_meta(self, dataset, **kwd): + # AGPFile reads and validates entire file. + AGPFile(dataset.file_name) + super().set_meta(dataset, **kwd) + + def sniff_prefix(self, file_prefix): + """ + Checks for and does cursory validation on data that looks like AGP + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('eg1.agp') + >>> GoldenPath().sniff(fname) + True + >>> fname = get_test_fname('eg2.agp') + >>> GoldenPath().sniff(fname) + True + >>> fname = get_test_fname('1.bed') + >>> GoldenPath().sniff(fname) + False + >>> fname = get_test_fname('2.tabular') + >>> GoldenPath().sniff(fname) + False + """ + found_non_comment_lines = False + try: + for line in iter_headers(file_prefix, '\t', comment_designator='#'): + if line: + if len(line) != 9: + return False + assert line[4] in ['A', 'D', 'F', 'G', 'O', 'P', 'W', 'N', 'U'] + ostensible_numbers = line[1:3] + if line[4] in ['U', 'N']: + ostensible_numbers.append(line[5]) + assert line[6] in ['scaffold', 'contig', 'centromere', 'short_arm', 'heterochromatin', 'telomere', 'repeat'] + assert line[7] in ['yes', 'no'] + assert line[8] in ['na', 'paired-ends', 'align_genus', 'align_xgenus', 'align_trnscript', 'within_clone', 'clone_contig', 'map', 'strobe', 'unspecified'] + else: + ostensible_numbers.extend([line[6], line[7]]) + assert line[8] in ['+', '-', '?', '0', 'na'] + if line[4] == 'U': + assert int(line[5]) == 100 + assert all(map(lambda x: str(x).isnumeric() and int(x) > 0, ostensible_numbers)) + found_non_comment_lines = True + except Exception: + return False + return found_non_comment_lines class AGPError(Exception): - """ Exception raised for AGP related errors. """ + """Exception raised for AGP related errors.""" def __init__(self, fname, line_number, message="Error in AGP file."): self.fname = fname @@ -43,7 +82,6 @@ class AGPError(Exception): class AGPFile: - """ A class storing the contents of an AGP v2.1 file. https://www.ncbi.nlm.nih.gov/assembly/agp/AGP_Specification/ @@ -58,9 +96,7 @@ class AGPFile: "pid": AGP part number """ - def __init__(self, in_file, mode="r"): - if mode != "r" and mode != "w": - raise ValueError("AGPFile mode must be either read ('r') or write ('w').") + def __init__(self, in_file): self._agp_version = "2.1" self._fname = os.path.abspath(in_file) @@ -74,12 +110,7 @@ class AGPFile: self._seen_objs = set() # Read the contents of the AGP file - if mode == "r": - self._read_file() - - def __iter__(self): - for obj in self._objects: - yield str(obj.obj), [dict(agp_line) for agp_line in obj] + self._read_file() def _read_file(self): """ @@ -163,21 +194,6 @@ class AGPFile: """ Calculate the number of lines in the current state of the AGP file. """ return sum([len(self._comment_lines)] + [obj.num_lines for obj in self._objects]) - @property - def num_objs(self): - return len(self._objects) - - def add_pragma(self): - pragma = f"## agp-version {self.agp_version}" - if self._comment_lines: - new_comment_lines = [pragma] - for i in self._comment_lines: - if i != pragma: - new_comment_lines.append(i) - self._comment_lines = new_comment_lines - else: - self._comment_lines.append(pragma) - def iterate_objs(self): """ Iterate over the objects of the AGP file. """ for obj in self._objects: @@ -189,75 +205,10 @@ class AGPFile: for j in obj.iterate_lines(): yield j - def add_comment(self, c): - if not isinstance(c, str): - raise TypeError("Comment must be a string") - - if not c.startswith("#"): - raise ValueError("Comment must start with a '#' character") - - if c not in self._comment_lines: - self._comment_lines.append(c) - - def add_seq_line(self, obj, obj_beg, obj_end, pid, comp_type, comp, comp_beg, comp_end, orientation): - """ - # TODO fill this out - :param obj: - :param obj_beg: - :param obj_end: - :param pid: - :param comp_type: - :param comp: - :param comp_beg: - :param comp_end: - :param orientation: - """ - line_number = self.num_lines + 1 - agp_line = AGPSeqLine(self.fname, line_number, obj, obj_beg, obj_end, pid, comp_type, comp, comp_beg, comp_end, orientation) - self._add_line(agp_line) - - def add_gap_line(self, obj, obj_beg, obj_end, pid, comp_type, gap_len, gap_type, linkage, linkage_evidence): - """ - # TODO fill this out - :param obj: - :param obj_beg: - :param obj_end: - :param pid: - :param comp_type: - :param gap_len: - :param gap_type: - :param linkage: - :param linkage_evidence: - """ - line_number = self.num_lines + 1 - agp_line = AGPGapLine(self.fname, line_number, obj, obj_beg, obj_end, pid, comp_type, gap_len, gap_type, linkage, linkage_evidence) - self._add_line(agp_line) - - def pop_agp_line(self): - """ Remove the last AGP line and update state info accordingly. """ - if not self._objects: - return - - if self._objects[-1].num_lines == 1: - self._seen_objs.remove(self._objects[-1].obj) - self._objects = self._objects[:-1] - - else: - self._objects[-1].pop_line() - - def write(self): - """ Write the agp contents to a file. """ - with open(self.fname, "w") as f: - if self._comment_lines: - f.write("\n".join(self._comment_lines) + "\n") - if self._objects: - f.write("\n".join([str(obj) for obj in self._objects]) + "\n") - class AGPObject: - """ - This (python) object represents an AGP object. Objects will consist of AGP lines, and have to adhere to + Represents an AGP object. Objects will consist of AGP lines, and have to adhere to certain rules. By organizing AGP lines into the objects that they comprise, we can easily calculate stats about the assembly (the collection of objects). """ @@ -331,7 +282,6 @@ class AGPObject: class AGPLine(object, metaclass=abc.ABCMeta): - """ An abstract base class representing a single AGP file line. Inheriting subclasses should override or implement new methods to check the validity of a single AFP line. Validity diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index cb6eb4ece88..2716e1ceea3 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -460,60 +460,6 @@ class Taxonomy(Tabular): return self.make_html_table(dataset, column_names=self.column_names) -@dataproviders.decorators.has_dataproviders -@build_sniff_from_prefix -class GoldenPath(Tabular): - """Class describing NCBI's Golden Path assembly format""" - edam_format = 'format_3693' - file_ext = 'agp' - - @dataproviders.decorators.dataprovider_factory('dict', dataproviders.column.GoldenPathDataProvider.settings) - def dict_dataprovider(self, dataset, **settings): - return super().dict_dataprovider(dataset, **settings) - - def sniff_prefix(self, file_prefix): - """ - Checks for and does cursory validation on data that looks like AGP - - >>> from galaxy.datatypes.sniff import get_test_fname - >>> fname = get_test_fname('eg1.agp') - >>> GoldenPath().sniff(fname) - True - >>> fname = get_test_fname('eg2.agp') - >>> GoldenPath().sniff(fname) - True - >>> fname = get_test_fname('1.bed') - >>> GoldenPath().sniff(fname) - False - >>> fname = get_test_fname('2.tabular') - >>> GoldenPath().sniff(fname) - False - """ - found_non_comment_lines = False - try: - for line in iter_headers(file_prefix, '\t', comment_designator='#'): - if line and not line[0].startswith('#'): - if len(line) != 9: - return False - assert line[4] in ['A', 'D', 'F', 'G', 'O', 'P', 'W', 'N', 'U'] - ostensible_numbers = line[1:3] - if line[4] in ['U', 'N']: - ostensible_numbers.append(line[5]) - assert line[6] in ['scaffold', 'contig', 'centromere', 'short_arm', 'heterochromatin', 'telomere', 'repeat'] - assert line[7] in ['yes', 'no'] - assert line[8] in ['na', 'paired-ends', 'align_genus', 'align_xgenus', 'align_trnscript', 'within_clone', 'clone_contig', 'map', 'strobe', 'unspecified'] - else: - ostensible_numbers.extend([line[6], line[7]]) - assert line[8] in ['+', '-', '?', '0', 'na'] - if line[4] == 'U': - assert int(line[5]) == 100 - assert all(map(lambda x: str(x).isnumeric() and int(x) > 0, ostensible_numbers)) - found_non_comment_lines = True - except Exception: - return False - return found_non_comment_lines - - @dataproviders.decorators.has_dataproviders @build_sniff_from_prefix class Sam(Tabular): diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 6e5c682491c..3393db81073 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -17,8 +17,6 @@ from bx.bbi.bigbed_file import BigBedFile from bx.bbi.bigwig_file import BigWigFile from bx.interval_index_file import Indexes -from galaxy.datatypes.dataproviders import column -from galaxy.datatypes.goldenpath import AGPFile from galaxy.datatypes.interval import Bed, Gff, Gtf from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFFeature, GFFInterval, GFFReaderWrapper, parse_gff_attributes from galaxy.visualization.data_providers.basic import BaseDataProvider @@ -314,26 +312,6 @@ class FilterableMixin: return filters -class GoldenPathDataProvider(column.GoldenPathDataProvider): - ''' - Class that defines a data provider for AGP datasets. - ''' - dataset_type: dict - - def __init__(self, original_dataset, **kwargs): - filename = original_dataset.dataset.get_file_name() - self.goldenpath = AGPFile(filename) - self.columns = [str(gp) for gp in self.goldenpath] - super().__init__(filename, columns=self.columns, **kwargs) - - def __iter__(self): - for gp in self.goldenpath: - yield gp - - def get_data(self, columns=None): - return dict(self.goldenpath) - - class TabixDataProvider(GenomeDataProvider, FilterableMixin): dataset_type = 'tabix' diff --git a/lib/galaxy/visualization/data_providers/registry.py b/lib/galaxy/visualization/data_providers/registry.py index 760acad21e3..ee3a5aa2c71 100644 --- a/lib/galaxy/visualization/data_providers/registry.py +++ b/lib/galaxy/visualization/data_providers/registry.py @@ -7,7 +7,7 @@ from galaxy.datatypes.interval import ( Gtf, Interval ) -from galaxy.datatypes.tabular import GoldenPath, Tabular, Vcf +from galaxy.datatypes.tabular import Tabular, Vcf from galaxy.datatypes.xml import Phyloxml from galaxy.model import NoConverterException from galaxy.visualization.data_providers import genome @@ -35,12 +35,10 @@ class DataProviderRegistry: "default": genome.TabixDataProvider }, "interval_index": genome.IntervalIndexDataProvider, - "agp": genome.GoldenPathDataProvider, "bai": genome.BamDataProvider, "bam": genome.SamDataProvider, "bigwig": genome.BigWigDataProvider, "bigbed": genome.BigBedDataProvider, - "column_with_stats": ColumnDataProvider } @@ -51,6 +49,8 @@ class DataProviderRegistry: """ data_provider = None + # any datatype class that is a subclass of another needs to be + # checked before the parent in this conditional. if raw: # Working with raw data. if isinstance(original_dataset.datatype, Gff): @@ -59,13 +59,6 @@ class DataProviderRegistry: data_provider_class = genome.RawBedDataProvider elif isinstance(original_dataset.datatype, Vcf): data_provider_class = genome.RawVcfDataProvider - # Quick reminder, to save the next developer a few minutes of - # debugging time, that any datatype class that is a subclass of - # another needs to be checked before the parent in this conditional. - elif isinstance(original_dataset.datatype, GoldenPath): - data_provider_class = genome.GoldenPathDataProvider - # Otherwise GoldenPath, being a subclass of Tabular, would match - # this clause and we would return the wrong data provider class. elif isinstance(original_dataset.datatype, Tabular): data_provider_class = ColumnDataProvider elif isinstance(original_dataset.datatype, (Nexus, Newick, Phyloxml)): From e74f44d46d9f66bb69180e1dd874f53097dbcdf8 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 10 Nov 2021 15:25:58 +0100 Subject: [PATCH 032/781] Update cwltool pin to 3.1.20211107152837 --- lib/galaxy/dependencies/dev-requirements.txt | 13 ++++++------- .../dependencies/pinned-lint-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 13 ++++++------- pyproject.toml | 2 +- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 4657f237f2e..0b59def4a7a 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -50,7 +50,7 @@ commonmark==0.9.1; python_version >= "3.6" and python_version < "4.0" contextvars==2.4; python_version < "3.7" and python_version >= "3.6" coverage==6.1.1; python_version >= "3.6" cryptography==35.0.0; python_version >= "3.6" -cwltool==3.0.20201109103151; python_version >= "3.6" and python_version < "4" +cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4" dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.6.1" debtcollector==2.3.0; python_version >= "3.6" decorator==5.1.0; python_version >= "3.6" @@ -193,8 +193,7 @@ pytz==2021.3; python_version >= "3.6" and python_full_version < "3.0.0" or pytho pyuwsgi==2.0.19.1.post0 pyyaml==6.0; python_version >= "3.6" pyzmq==22.3.0; python_version >= "3.6" -rdflib-jsonld==0.5.0; python_version >= "3.6" and python_version < "4" -rdflib==4.2.2; python_version >= "3.6" and python_version < "4" +rdflib==5.0.0; python_version >= "3.6" and python_version < "4" recommonmark==0.7.1 refgenconf==0.12.2 repoze.lru==0.7 @@ -207,13 +206,13 @@ rfc3986==1.5.0; python_version >= "3.6" rich==10.11.0; python_version >= "3.6" and python_version < "4.0" routes==2.5.1 rsa==4.7.2; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") -ruamel.yaml.clib==0.2.6; platform_python_implementation == "CPython" and python_version < "3.8" and python_version >= "3.6" -ruamel.yaml==0.16.5; python_version >= "3.6" and python_version < "4" +ruamel.yaml.clib==0.2.6; platform_python_implementation == "CPython" and python_version < "3.10" and python_version >= "3.6" +ruamel.yaml==0.17.17; python_version >= "3.6" and python_version < "4" s3transfer==0.5.0; python_version >= "3.6" -schema-salad==7.1.20210611090601; python_version >= "3.6" and python_version < "4" +schema-salad==8.2.20211104054942; python_version >= "3.6" and python_version < "4" selenium==3.141.0 setuptools-scm==5.0.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" -shellescape==3.4.1; python_version >= "3.6" and python_version < "4" +shellescape==3.8.1; python_version >= "3.6" and python_version < "4" simplejson==3.17.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" sniffio==1.2.0; python_version >= "3.6" diff --git a/lib/galaxy/dependencies/pinned-lint-requirements.txt b/lib/galaxy/dependencies/pinned-lint-requirements.txt index ccdf98a61c0..4d84fe9b4ee 100644 --- a/lib/galaxy/dependencies/pinned-lint-requirements.txt +++ b/lib/galaxy/dependencies/pinned-lint-requirements.txt @@ -21,6 +21,6 @@ types-paramiko==2.7.4 types-pkg-resources==0.1.3 types-python-dateutil==2.8.2 types-PyYAML==6.0.0 -types-requests==2.25.12 +types-requests==2.26.0 types-six==1.16.2 typing-extensions==3.10.0.2 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 8a4c76b3817..4c554791d2a 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -47,7 +47,7 @@ coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" a commonmark==0.9.1; python_version >= "3.6" and python_version < "4.0" contextvars==2.4; python_version >= "3.6" and python_version < "3.7" cryptography==35.0.0; python_version >= "3.6" -cwltool==3.0.20201109103151; python_version >= "3.6" and python_version < "4" +cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4" dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.6.1" debtcollector==2.3.0; python_version >= "3.6" decorator==5.1.0; python_version >= "3.6" @@ -164,8 +164,7 @@ pytz==2021.3; python_version >= "3.6" and python_full_version < "3.0.0" or pytho pyuwsgi==2.0.19.1.post0 pyyaml==6.0; python_version >= "3.6" pyzmq==22.3.0; python_version >= "3.6" -rdflib-jsonld==0.5.0; python_version >= "3.6" and python_version < "4" -rdflib==4.2.2; python_version >= "3.6" and python_version < "4" +rdflib==5.0.0; python_version >= "3.6" and python_version < "4" refgenconf==0.12.2 repoze.lru==0.7 requests-oauthlib==1.3.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" @@ -176,12 +175,12 @@ rfc3986==1.5.0; python_version >= "3.6" rich==10.11.0; python_version >= "3.6" and python_version < "4.0" routes==2.5.1 rsa==4.7.2; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") -ruamel.yaml.clib==0.2.6; platform_python_implementation == "CPython" and python_version < "3.8" and python_version >= "3.6" -ruamel.yaml==0.16.5; python_version >= "3.6" and python_version < "4" +ruamel.yaml.clib==0.2.6; platform_python_implementation == "CPython" and python_version < "3.10" and python_version >= "3.6" +ruamel.yaml==0.17.17; python_version >= "3.6" and python_version < "4" s3transfer==0.5.0; python_version >= "3.6" -schema-salad==7.1.20210611090601; python_version >= "3.6" and python_version < "4" +schema-salad==8.2.20211104054942; python_version >= "3.6" and python_version < "4" setuptools-scm==5.0.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" -shellescape==3.4.1; python_version >= "3.6" and python_version < "4" +shellescape==3.8.1; python_version >= "3.6" and python_version < "4" simplejson==3.17.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" social-auth-core==3.3.0 diff --git a/pyproject.toml b/pyproject.toml index ef2c4afa529..558373c92ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ cloudauthz = "==0.6.0" cloudbridge = "*" contextvars = {version = "*", python = "~3.6"} circus = "*" -cwltool = "==3.0.20201109103151" +cwltool = "==3.1.20211107152837" dictobj = "*" docutils = "!=0.17, !=0.17.1" edam-ontology = "*" From 046f495e4aca26de38ca3674a6d4b013c4328453 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 10 Nov 2021 09:42:58 -0500 Subject: [PATCH 033/781] Update lib/galaxy/webapps/galaxy/api/history_contents.py Co-authored-by: Marius van den Beek --- lib/galaxy/webapps/galaxy/api/history_contents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index c03b01ea011..5a1d6343f4a 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -518,7 +518,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, trans, history_id, serialization_params, filter_params, hid, limit, since, ) - @expose_api_raw_anonymous + @expose_api_anonymous def contents_after(self, trans, history_id, hid, limit, **kwd): """ Return {limit} history items with hid > {hid}. From 29d7705bae85f9aa9f333a7c0812a6e53b8ccd44 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 10 Nov 2021 09:43:16 -0500 Subject: [PATCH 034/781] Update lib/galaxy/webapps/galaxy/services/history_contents.py Co-authored-by: Marius van den Beek --- lib/galaxy/webapps/galaxy/services/history_contents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index bb184e28300..63406ea9ae9 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -788,7 +788,7 @@ class HistoriesContentsService(ServiceBase): trans.response.headers['history_size'] = str(history.disk_size) trans.response.headers['history_empty'] = json.dumps(history.empty) # convert to proper bool - return json.dumps(contents) + return contents def _hid_greater_than(self, hid: int) -> HistoryContentsFilterList: return [["hid", "gt", hid]] From c2b5a58c627ce6e5ae10c63f86f534665bd61265 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 10 Nov 2021 10:27:08 -0500 Subject: [PATCH 035/781] Replace after with before --- .../webapps/galaxy/api/history_contents.py | 6 +++--- lib/galaxy/webapps/galaxy/buildapp.py | 4 ++-- .../galaxy/services/history_contents.py | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 5a1d6343f4a..44f5c4d7fde 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -519,18 +519,18 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, ) @expose_api_anonymous - def contents_after(self, trans, history_id, hid, limit, **kwd): + def contents_before(self, trans, history_id, hid, limit, **kwd): """ Return {limit} history items with hid > {hid}. - GET /api/histories/{history_id}/contents/after/{hid}/{limit} + GET /api/histories/{history_id}/contents/before/{hid}/{limit} """ serialization_params = parse_serialization_params(default_view='betawebclient', **kwd) filter_params = self._parse_rest_params(kwd) hid = int(hid) limit = int(limit) - return self.service.contents_after( + return self.service.contents_before( trans, history_id, serialization_params, filter_params, hid, limit ) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 8e073df6a2e..726f82a4d3a 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -317,8 +317,8 @@ def populate_api_routes(webapp, app): action="contents_near", controller='history_contents', conditions=dict(method=["GET"])) - webapp.mapper.connect("/api/histories/{history_id}/contents/after/{hid}/{limit}", - action="contents_after", + webapp.mapper.connect("/api/histories/{history_id}/contents/before/{hid}/{limit}", + action="contents_before", controller='history_contents', conditions=dict(method=["GET"])) webapp.mapper.resource('user', diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 63406ea9ae9..68f55086e2d 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -756,7 +756,7 @@ class HistoriesContentsService(ServiceBase): return json.dumps(contents) - def contents_after( + def contents_before( self, trans, history_id: EncodedDatabaseIdField, serialization_params: SerializationParams, @@ -771,18 +771,17 @@ class HistoriesContentsService(ServiceBase): history: History = self.history_manager.get_accessible( self.decode_id(history_id), trans.user, current_history=trans.history ) - up_params = filter_params + self._hid_greater_than(hid) - up_order = 'hid-asc' - contents_up, up_count = self._seek(history, up_params, up_order, limit, serialization_params) + params = filter_params + self._hid_less_than(hid) + order = 'hid-dsc' + contents, count = self._seek(history, params, order, limit, serialization_params) min_hid, max_hid = self._get_filtered_extrema(history, filter_params) - contents = self._expand_contents(trans, contents_up, serialization_params) - contents.reverse() + contents = self._expand_contents(trans, contents, serialization_params) # Put stats in http headers - trans.response.headers['matches_up'] = len(contents_up) - trans.response.headers['total_matches_up'] = up_count + trans.response.headers['matches_down'] = len(contents) + trans.response.headers['total_matches_down'] = count trans.response.headers['max_hid'] = max_hid trans.response.headers['min_hid'] = min_hid trans.response.headers['history_size'] = str(history.disk_size) @@ -793,6 +792,9 @@ class HistoriesContentsService(ServiceBase): def _hid_greater_than(self, hid: int) -> HistoryContentsFilterList: return [["hid", "gt", hid]] + def _hid_less_than(self, hid: int) -> HistoryContentsFilterList: + return [["hid", "lt", hid]] + def _hid_less_or_equal_than(self, hid: int) -> HistoryContentsFilterList: return [["hid", "le", hid]] From 7b86b5021576146df3496827506f9ab702758521 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 10 Nov 2021 10:17:46 -0500 Subject: [PATCH 036/781] Rev pulsar-galaxy-lib. --- lib/galaxy/dependencies/dev-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 4657f237f2e..72423bbc728 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -147,7 +147,7 @@ prompt-toolkit==3.0.19; python_full_version >= "3.6.1" and python_version >= "3. protobuf==3.19.1; python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.10 +pulsar-galaxy-lib==0.14.12 py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.5.0" and python_version >= "3.6" and implementation_name == "pypy" pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 8a4c76b3817..ebe59917922 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -131,7 +131,7 @@ prompt-toolkit==3.0.19; python_full_version >= "3.6.1" and python_version >= "3. protobuf==3.19.1; python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.10 +pulsar-galaxy-lib==0.14.12 py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.5.0" pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") diff --git a/pyproject.toml b/pyproject.toml index ef2c4afa529..defdb777272 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ paramiko = "*" Parsley = "*" Paste = "*" psutil = "*" -pulsar-galaxy-lib = ">=0.14.9" +pulsar-galaxy-lib = ">=0.14.12" pycryptodome = "*" pykwalify = "*" pyparsing = "*" From e356cd0b6473e09897571da0b62014513684c462 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Mon, 8 Nov 2021 15:10:37 +0100 Subject: [PATCH 037/781] implement input color --- .../components/Form/Elements/FormColor.vue | 28 +++++++++++++++++++ client/src/components/Form/FormElement.vue | 3 ++ 2 files changed, 31 insertions(+) create mode 100644 client/src/components/Form/Elements/FormColor.vue diff --git a/client/src/components/Form/Elements/FormColor.vue b/client/src/components/Form/Elements/FormColor.vue new file mode 100644 index 00000000000..03ec904ea38 --- /dev/null +++ b/client/src/components/Form/Elements/FormColor.vue @@ -0,0 +1,28 @@ + + + diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue index c2e0cf15f17..4f88562c2a0 100644 --- a/client/src/components/Form/FormElement.vue +++ b/client/src/components/Form/FormElement.vue @@ -28,6 +28,7 @@ :id="id" :info="attrs['info']" /> + Date: Mon, 8 Nov 2021 15:38:27 +0100 Subject: [PATCH 038/781] delete legacy code --- .../components/Form/Elements/parameters.js | 9 -- client/src/mvc/ui/ui-color-picker.js | 142 ------------------ client/src/style/scss/ui.scss | 59 -------- 3 files changed, 210 deletions(-) delete mode 100644 client/src/mvc/ui/ui-color-picker.js diff --git a/client/src/components/Form/Elements/parameters.js b/client/src/components/Form/Elements/parameters.js index 3b6d7cecb24..f2b56cece3d 100644 --- a/client/src/components/Form/Elements/parameters.js +++ b/client/src/components/Form/Elements/parameters.js @@ -10,7 +10,6 @@ import SelectLibrary from "mvc/ui/ui-select-library"; import SelectFtp from "mvc/ui/ui-select-ftp"; import RulesEdit from "mvc/ui/ui-rules-edit"; import FileSource from "mvc/ui/ui-file-source"; -import ColorPicker from "mvc/ui/ui-color-picker"; import DataPicker from "mvc/ui/ui-data-picker"; // create form view @@ -185,14 +184,6 @@ export default Backbone.View.extend({ }); }, - /** Color picker field */ - _fieldColor: function (input_def) { - return new ColorPicker({ - id: `field-${input_def.id}`, - onchange: input_def.onchange, - }); - }, - /** Data dialog picker field */ _fieldDialog: function (input_def) { return new DataPicker({ diff --git a/client/src/mvc/ui/ui-color-picker.js b/client/src/mvc/ui/ui-color-picker.js deleted file mode 100644 index e862d974f26..00000000000 --- a/client/src/mvc/ui/ui-color-picker.js +++ /dev/null @@ -1,142 +0,0 @@ -/** Renders the color picker used e.g. in the tool form **/ -import $ from "jquery"; -import Backbone from "backbone"; -import Utils from "utils/utils"; - -function hex(x) { - return `0${parseInt(x).toString(16)}`.slice(-2); -} - -export default Backbone.View.extend({ - colors: { - standard: ["c00000", "ff0000", "ffc000", "ffff00", "92d050", "00b050", "00b0f0", "0070c0", "002060", "7030a0"], - base: ["ffffff", "000000", "eeece1", "1f497d", "4f81bd", "c0504d", "9bbb59", "8064a2", "4bacc6", "f79646"], - theme: [ - ["f2f2f2", "7f7f7f", "ddd9c3", "c6d9f0", "dbe5f1", "f2dcdb", "ebf1dd", "e5e0ec", "dbeef3", "fdeada"], - ["d8d8d8", "595959", "c4bd97", "8db3e2", "b8cce4", "e5b9b7", "d7e3bc", "ccc1d9", "b7dde8", "fbd5b5"], - ["bfbfbf", "3f3f3f", "938953", "548dd4", "95b3d7", "d99694", "c3d69b", "b2a2c7", "92cddc", "fac08f"], - ["a5a5a5", "262626", "494429", "17365d", "366092", "953734", "76923c", "5f497a", "31859b", "e36c09"], - ["7f7f7e", "0c0c0c", "1d1b10", "0f243e", "244061", "632423", "4f6128", "3f3151", "205867", "974806"], - ], - }, - - initialize: function (options) { - this.options = Utils.merge(options, {}); - this.setElement(this._template()); - this.$panel = this.$(".ui-color-picker-panel"); - this.$view = this.$(".ui-color-picker-view"); - this.$value = this.$(".ui-color-picker-value"); - this.$header = this.$(".ui-color-picker-header"); - this._build(); - this.visible = false; - this.value(this.options.value); - this.$boxes = this.$(".ui-color-picker-box"); - var self = this; - this.$boxes.on("click", function () { - self.value($(this).css("background-color")); - self.$header.trigger("click"); - }); - this.$header.on("click", () => { - self.visible = !self.visible; - if (self.visible) { - self.$view.fadeIn("fast"); - } else { - self.$view.fadeOut("fast"); - } - }); - }, - - /** Get/set value */ - value: function (new_val) { - if (new_val !== undefined && new_val !== null) { - this.$value.css("background-color", new_val); - this.$(".ui-color-picker-box").empty(); - this.$(this._getValue()).html(this._templateCheck()); - this.options.onchange && this.options.onchange(new_val); - } - return this._getValue(); - }, - - /** Get value from dom */ - _getValue: function () { - var rgb = this.$value.css("background-color"); - rgb = rgb.match(/^rgb\((\d+),\s*(\d+),\s*(\d+)\)$/); - if (rgb) { - return `#${hex(rgb[1])}${hex(rgb[2])}${hex(rgb[3])}`; - } else { - return null; - } - }, - - /** Build color panel */ - _build: function () { - this._content({ colors: this.colors.base }); - for (var i in this.colors.theme) { - var line_def = {}; - if (i == 0) { - line_def["bottom"] = true; - } else if (i != this.colors.theme.length - 1) { - line_def["top"] = true; - line_def["bottom"] = true; - } else { - line_def["top"] = true; - } - line_def["colors"] = this.colors.theme[i]; - this._content(line_def); - } - this._content({ colors: this.colors.standard }); - }, - - /** Create content */ - _content: function (options) { - var colors = options.colors; - var $content = $(this._templateContent()); - var $line = $content.find(".line"); - this.$panel.append($content); - for (var i in colors) { - var $box = $(this._templateBox(colors[i])); - if (options.top) { - $box.css("border-top", "none"); - $box.css("border-top-left-radius", "0px"); - $box.css("border-top-right-radius", "0px"); - } - if (options.bottom) { - $box.css("border-bottom", "none"); - $box.css("border-bottom-left-radius", "0px"); - $box.css("border-bottom-right-radius", "0px"); - } - $line.append($box); - } - return $content; - }, - - /** Check icon */ - _templateCheck: function () { - return '
'; - }, - - /** Content template */ - _templateContent: function () { - return '
'; - }, - - /** Box template */ - _templateBox: function (color) { - return `
`; - }, - - /** Main template */ - _template: function () { - return ` -
-
-
-
Select a color
-
-
-
-
-
- `; - }, -}); diff --git a/client/src/style/scss/ui.scss b/client/src/style/scss/ui.scss index bce497b38ca..750a97b7024 100644 --- a/client/src/style/scss/ui.scss +++ b/client/src/style/scss/ui.scss @@ -353,65 +353,6 @@ $ui-margin-horizontal-large: $margin-v * 2; } } -.ui-color-picker { - .ui-color-picker-header { - cursor: pointer; - } - .ui-color-picker-value { - float: left; - border-radius: 3px; - border: solid 1px #000000; - width: 12px; - height: 12px; - margin-right: 5px; - } - .ui-color-picker-label { - line-height: 1.2em; - margin-bottom: 5px; - } - .ui-color-picker-view { - height: 100%; - overflow: auto; - display: none; - float: left; - margin-bottom: 5px; - .ui-color-picker-panel { - width: 210px; - .ui-color-picker-content { - margin-bottom: 15px; - .badge { - padding-bottom: 2px; - } - .line { - .ui-color-picker-box { - cursor: pointer; - float: left; - margin-right: 5px; - border: solid 1px #c0c0c0; - width: 15px; - height: 15px; - border-radius: 2px; - .ui-color-picker-check { - color: black; - font-size: 1.2em; - position: relative; - left: 1px; - } - } - } - } - .ui-color-picker-content:first-child { - padding-top: 5px; - padding-bottom: 10px; - } - .ui-color-picker-content:last-child { - padding-top: 85px; - padding-bottom: 5px; - } - } - } -} - .ui-select { position: relative; .icon-dropdown { From 98b281d943c629eadb7e9ea91935c35946118e54 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Mon, 8 Nov 2021 16:36:05 +0100 Subject: [PATCH 039/781] add label, make cursor pointer --- client/src/components/Form/Elements/FormColor.vue | 14 ++++++++++++-- .../LibraryFolder/TopToolbar/FolderTopBar.vue | 2 +- .../LibraryFolder/library-folder-table.css | 2 +- .../LibraryFolder/TopToolbar => utils}/pointer.css | 0 4 files changed, 14 insertions(+), 4 deletions(-) rename client/src/{components/Libraries/LibraryFolder/TopToolbar => utils}/pointer.css (100%) diff --git a/client/src/components/Form/Elements/FormColor.vue b/client/src/components/Form/Elements/FormColor.vue index 03ec904ea38..67ff263872a 100644 --- a/client/src/components/Form/Elements/FormColor.vue +++ b/client/src/components/Form/Elements/FormColor.vue @@ -1,7 +1,10 @@ @@ -13,6 +16,10 @@ export default { type: String, required: true, }, + id: { + type: String, + required: true, + }, }, computed: { currentValue: { @@ -26,3 +33,6 @@ export default { }, }; + \ No newline at end of file diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue index fabf9dfeba7..d3b38b2d980 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -385,5 +385,5 @@ export default { diff --git a/client/src/components/Libraries/LibraryFolder/library-folder-table.css b/client/src/components/Libraries/LibraryFolder/library-folder-table.css index 6c57698b8c2..ac16c6be460 100644 --- a/client/src/components/Libraries/LibraryFolder/library-folder-table.css +++ b/client/src/components/Libraries/LibraryFolder/library-folder-table.css @@ -1,4 +1,4 @@ -@import "./TopToolbar/pointer.css"; +@import "utils/pointer.css"; th:focus { outline: none !important; diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/pointer.css b/client/src/utils/pointer.css similarity index 100% rename from client/src/components/Libraries/LibraryFolder/TopToolbar/pointer.css rename to client/src/utils/pointer.css From 3eb0dd36bfbd2bef0eac9b5f29efbbd2ab8093ea Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Mon, 8 Nov 2021 16:56:19 +0100 Subject: [PATCH 040/781] fix styling --- client/src/components/Form/Elements/FormColor.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/components/Form/Elements/FormColor.vue b/client/src/components/Form/Elements/FormColor.vue index 67ff263872a..caf576c3275 100644 --- a/client/src/components/Form/Elements/FormColor.vue +++ b/client/src/components/Form/Elements/FormColor.vue @@ -1,5 +1,5 @@