From d69a6fcd0e119238b4812abd1bc6bfa517195142 Mon Sep 17 00:00:00 2001 From: rikeshi Date: Sat, 13 Feb 2021 11:50:17 +0100 Subject: [PATCH 01/10] remove kwds arg from call to get_stored_accessible_workflow fix 'key' causing TypeError on GET request --- lib/galaxy/webapps/galaxy/api/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 214a5fe1bb9..f09e5a46ba5 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -244,7 +244,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn Lists all versions of this workflow. """ - stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id, **kwds) + stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id) return [{'version': i, 'update_time': str(w.update_time), 'steps': len(w.steps)} for i, w in enumerate(reversed(stored_workflow.workflows))] @expose_api From 9cc01de07a6478113dc8ca6ff379bce42893c55e Mon Sep 17 00:00:00 2001 From: rikeshi Date: Mon, 15 Feb 2021 17:22:30 +0000 Subject: [PATCH 02/10] Apply suggestions from code review Co-authored-by: Simon Bray <32272674+simonbray@users.noreply.github.com> --- lib/galaxy/webapps/galaxy/api/workflows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index f09e5a46ba5..2b9ef427d2d 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -244,7 +244,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn Lists all versions of this workflow. """ - stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id) + instance = util.string_as_bool(kwds.get("instance", "false")) + stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id, by_stored_id=not instance) return [{'version': i, 'update_time': str(w.update_time), 'steps': len(w.steps)} for i, w in enumerate(reversed(stored_workflow.workflows))] @expose_api From e2076fbb9dd7c79913a4c696d9c1eed5ec031add Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Feb 2021 18:08:31 +0100 Subject: [PATCH 03/10] fix genration of config files --- lib/galaxy/tools/parameters/wrapped_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index b65063c7262..a0574981292 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -97,7 +97,7 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"): if input.multiple and packaging.version.parse(str(profile)) >= packaging.version.parse('20.05'): json_value = [_ for _ in _cast_if_not_none(value_wrapper.value, list)] else: - json_value = _cast_if_not_none(value_wrapper, str) + json_value = _cast_if_not_none(value_wrapper.value, str) elif input_type == "data_column": # value is a SelectToolParameterWrapper() if input.multiple: From bd5e8f326cc63efe3a2711836c10a57d88b9ca9d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Feb 2021 19:22:29 +0100 Subject: [PATCH 04/10] fix profile logic for configfile generation some follow up fixes to https://github.com/galaxyproject/galaxy/pull/9279/files https://github.com/galaxyproject/galaxy/pull/9776/files - wrong profile for the profile test tool (it used 20.01, but should have used 20.05) - the non-profile test tool used wrong assumptions. now fixed and also tested against 19.09 - in the profile version `value_wrapper.value` needs to be casted for `multiple==False` otherwise we gat `"None"` (which I do not understand 100%) --- lib/galaxy/tools/parameters/wrapped_json.py | 9 ++++++--- test/functional/tools/inputs_as_json.xml | 6 +++--- test/functional/tools/inputs_as_json_profile.xml | 8 ++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index a0574981292..9a3393ffc21 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -94,10 +94,13 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"): elif input_type == "boolean": json_value = _cast_if_not_none(value_wrapper, bool) elif input_type == "select": - if input.multiple and packaging.version.parse(str(profile)) >= packaging.version.parse('20.05'): - json_value = [_ for _ in _cast_if_not_none(value_wrapper.value, list)] + if packaging.version.parse(str(profile)) < packaging.version.parse('20.05'): + json_value = _cast_if_not_none(value_wrapper, str) else: - json_value = _cast_if_not_none(value_wrapper.value, str) + if input.multiple: + json_value = [str(_) for _ in _cast_if_not_none(value_wrapper.value, list)] + else: + json_value = _cast_if_not_none(value_wrapper.value, str) elif input_type == "data_column": # value is a SelectToolParameterWrapper() if input.multiple: diff --git a/test/functional/tools/inputs_as_json.xml b/test/functional/tools/inputs_as_json.xml index 465aba5eda2..1aff602e796 100644 --- a/test/functional/tools/inputs_as_json.xml +++ b/test/functional/tools/inputs_as_json.xml @@ -27,8 +27,8 @@ if test_case == "1": assert_equals(as_dict["inttest"], 12456) assert_equals(as_dict["floattest"], 6.789) assert_equals(as_dict["radio_select"], "a_radio") - assert_equals(as_dict["optional_select"], None) - assert_equals(as_dict["optional_multiple_select"], []) + assert_equals(as_dict["optional_select"], "None") + assert_equals(as_dict["optional_multiple_select"], "None") assert_equals(as_dict["repeat"][0]["r"], "000000") assert_equals(as_dict["repeat"][1]["r"], "FFFFFF") assert_equals(as_dict["cond"]["more_text"], "fdefault") @@ -43,7 +43,7 @@ elif test_case == "2": assert_equals(as_dict["floattest"], 1.0) assert_equals(as_dict["radio_select"], "a_radio") assert_equals(as_dict["optional_select"], "a") - assert_equals(as_dict["optional_multiple_select"], ['a', 'b']) + assert_equals(as_dict["optional_multiple_select"], 'a,b') assert_equals(as_dict["repeat"][0]["r"], "000000") assert_equals(as_dict["cond"]["cond_test"], "second") assert_equals(as_dict["cond"]["more_text"], "sdefault") diff --git a/test/functional/tools/inputs_as_json_profile.xml b/test/functional/tools/inputs_as_json_profile.xml index 9fc04164878..162a6fa7345 100644 --- a/test/functional/tools/inputs_as_json_profile.xml +++ b/test/functional/tools/inputs_as_json_profile.xml @@ -1,7 +1,7 @@ - + Date: Fri, 15 Jan 2021 19:06:56 +0000 Subject: [PATCH 05/10] Sync social-auth-core requirement in data package v4.0.2 dropped support for Python 3.5 --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- packages/data/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 762cfc3a441..27fcd08ea2c 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -82,7 +82,7 @@ pyparsing = "*" "Fabric3" = "*" paramiko = "*" cloudbridge = "*" -social_auth_core = {version = "==3.3.0", extras = ['openidconnect']} +social-auth-core = {version = "==3.3.0", extras = ['openidconnect']} cloudauthz = "==0.6.0" gxformat2 = "*" refgenconf = ">=0.7.0" diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 98424dcbef5..8753dc5cddd 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -9,7 +9,7 @@ parsley numpy<=1.16 pycryptodome pysam -social_auth_core +social-auth-core[openidconnect]==3.3.0 SQLAlchemy sqlalchemy-migrate sqlalchemy-utils From fc1d74d1d3a7d8c9d5a07dd71a14ab8caaec1474 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 8 Feb 2021 11:19:41 +0100 Subject: [PATCH 06/10] Deploy postgresql and rabbitmq in kubernetes Setting up minikube stops the postgres and rabbitmq containers that are set up by the workflow. These are setup with custom docker networks, so it isn't trivial to restart them. Instead we just deploy them to minikube. --- .ci/minikube-test-setup/deployment.yaml | 32 +++++++++ .ci/minikube-test-setup/start_services.sh | 12 ++++ .github/workflows/integration.yaml | 85 +++++++++++++---------- 3 files changed, 92 insertions(+), 37 deletions(-) create mode 100644 .ci/minikube-test-setup/deployment.yaml create mode 100644 .ci/minikube-test-setup/start_services.sh diff --git a/.ci/minikube-test-setup/deployment.yaml b/.ci/minikube-test-setup/deployment.yaml new file mode 100644 index 00000000000..d7afe110c83 --- /dev/null +++ b/.ci/minikube-test-setup/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: testing + name: testing +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: test + template: + metadata: + labels: + app.kubernetes.io/name: test + spec: + containers: + - image: postgres:12 + name: postgres + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: postgres + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + value: postgres + - image: rabbitmq + name: rabbitmq + ports: + - containerPort: 5672 diff --git a/.ci/minikube-test-setup/start_services.sh b/.ci/minikube-test-setup/start_services.sh new file mode 100644 index 00000000000..1e860dadcb7 --- /dev/null +++ b/.ci/minikube-test-setup/start_services.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -ex + +SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") +kubectl apply -f "$SCRIPTDIR/deployment.yaml" +kubectl expose deployment testing --type=LoadBalancer --name=testing-service + +CLUSTER_IP=$(kubectl get service testing-service -o jsonpath='{.spec.clusterIP}') +GALAXY_TEST_DBURI="postgresql://postgres:postgres@${CLUSTER_IP}:5432/galaxy?client_encoding=utf-8" +GALAXY_TEST_AMQP_URL="amqp://${CLUSTER_IP}:5672)//" +export GALAXY_TEST_DBURI +export GALAXY_TEST_AMQP_URL diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 3bd0aaf4d69..c48e5346388 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -6,14 +6,15 @@ env: jobs: test: name: Test - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: [3.7] - subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)'] + python-version: ['3.7'] + subset: ['kubernetes'] services: postgres: - image: postgres:11 + image: postgres:13 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -25,36 +26,46 @@ jobs: ports: - 5672:5672 steps: - - name: Prune unused docker image, volumes and containers - run: docker system prune -a -f - - name: Clean dotnet folder for space - if: matrix.subset == 'kubernetes' - run: rm -Rf /usr/share/dotnet - - name: Setup Minikube - if: matrix.subset == 'kubernetes' - id: minikube - uses: CodingNagger/minikube-setup-action@v1.0.3 - with: - minikube-version: "1.9.0-0_amd64" - - name: Launch Minikube - if: matrix.subset == 'kubernetes' - run: eval ${{ steps.minikube.outputs.launcher }} - - name: Check pods - if: matrix.subset == 'kubernetes' - run: | - kubectl get pods - - uses: actions/checkout@v2 - with: - path: 'galaxy root' - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip dir - uses: actions/cache@v1 - id: pip-cache - with: - path: ~/.cache/pip - key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} - - name: Run tests - run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' - working-directory: 'galaxy root' + - name: Prune unused docker image, volumes and containers + run: docker system prune -a -f + - name: Clean dotnet folder for space + if: matrix.subset == 'kubernetes' + run: rm -Rf /usr/share/dotnet + - name: Setup Minikube + if: matrix.subset == 'kubernetes' + id: minikube + uses: CodingNagger/minikube-setup-action@v1.0.3 + with: + minikube-version: "1.9.0-0_amd64" + - name: Launch Minikube + if: matrix.subset == 'kubernetes' + run: eval ${{ steps.minikube.outputs.launcher }} + - name: Check pods + if: matrix.subset == 'kubernetes' + run: | + kubectl get pods + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip dir + uses: actions/cache@v1 + id: pip-cache + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Run tests + if: matrix.subset != 'kubernetes' + run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + working-directory: 'galaxy root' + - name: Run tests + if: matrix.subset == 'kubernetes' + run: 'source .ci/minikube-test-setup/start_services.sh && ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + working-directory: 'galaxy root' + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: Integration test results (${{ matrix.python-version }}, ${{ matrix.subset }}) + path: 'galaxy root/run_integration_tests.html' From a8adab100ccf38cfc2a14de3c798479f5531213e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 8 Feb 2021 13:08:45 +0100 Subject: [PATCH 07/10] Restore all integration subsets --- .github/workflows/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index c48e5346388..410af8045a9 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: python-version: ['3.7'] - subset: ['kubernetes'] + subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)'] services: postgres: image: postgres:13 From db1618fc6a3c7454ab2d1ed8af8ea6d38aeb66e7 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Mon, 8 Feb 2021 14:01:28 +0100 Subject: [PATCH 08/10] Use multiline yaml Co-authored-by: Nicola Soranzo --- .github/workflows/integration.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 410af8045a9..5cba3dba72b 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -62,7 +62,9 @@ jobs: working-directory: 'galaxy root' - name: Run tests if: matrix.subset == 'kubernetes' - run: 'source .ci/minikube-test-setup/start_services.sh && ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + run: | + . .ci/minikube-test-setup/start_services.sh + ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}" working-directory: 'galaxy root' - uses: actions/upload-artifact@v2 if: failure() From 26f731f060de198ec6ab1108f76cc2efa1e1dcf0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 19 Feb 2021 04:24:53 +0000 Subject: [PATCH 09/10] Fix sniffers of interval datatypes for files starting with many comments For files starting with >60 comment lines, the `sniff_prefix()` method of Bed, Gff, Gff3 and Gtf classes was returning `True` when actually all headers lines were skipped. Also: - Don't skip lines having more than one column just because the first column is empty. - Properly skip comment lines in `is_column_based()` - Add test file which would have been incorrectly sniffed as `gtf` before. It still isn't sniffed as `gff3`, but that's fine because it has too many comment lines for the default header prefix. --- lib/galaxy/datatypes/interval.py | 156 ++++++++++-------- lib/galaxy/datatypes/sniff.py | 29 ++-- .../test/too_many_comments_gff3.tabular | 65 ++++++++ 3 files changed, 165 insertions(+), 85 deletions(-) create mode 100644 lib/galaxy/datatypes/test/too_many_comments_gff3.tabular diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 89d771b42e6..20728b5d3d8 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -477,9 +477,9 @@ class Bed(Interval): if not get_headers(file_prefix, '\t', comment_designator='#', count=1): return False try: - headers = iter_headers(file_prefix, '\t', comment_designator='#') - for hdr in headers: - if hdr[0] == '': + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t', comment_designator='#'): + if not hdr or hdr == ['']: continue if len(hdr) < 3 or len(hdr) > 12: return False @@ -542,7 +542,8 @@ class Bed(Interval): return False if len(block_sizes) != block_count or len(block_starts) != block_count: return False - return True + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -818,28 +819,33 @@ class Gff(Tabular, _RemoteCallMixin): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: return False - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: - return False + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + int(hdr[4]) + except Exception: + return False + if hdr[5] != '.': try: - int(hdr[3]) - int(hdr[4]) + float(hdr[5]) except Exception: return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in data.valid_strand: - return False - if hdr[7] not in self.valid_gff_frame: - return False - return True + if hdr[6] not in data.valid_strand: + return False + if hdr[7] not in self.valid_gff_frame: + return False + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -953,37 +959,41 @@ class Gff3(Gff): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('3') >= 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('3') >= 0: return True - elif hdr and hdr[0].startswith('##gff-version') and hdr[0].find('3') < 0: + elif hdr[0].startswith('##gff-version') and hdr[0].find('3') < 0: return False - # Header comments may have been stripped, so inspect the data - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + except Exception: + if hdr[3] != '.': return False + try: + int(hdr[4]) + except Exception: + if hdr[4] != '.': + return False + if hdr[5] != '.': try: - int(hdr[3]) + float(hdr[5]) except Exception: - if hdr[3] != '.': - return False - try: - int(hdr[4]) - except Exception: - if hdr[4] != '.': - return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in self.valid_gff3_strand: return False - if hdr[7] not in self.valid_gff3_phase: - return False - parse_gff3_attributes(hdr[8]) - return True + if hdr[6] not in self.valid_gff3_strand: + return False + if hdr[7] not in self.valid_gff3_phase: + return False + parse_gff3_attributes(hdr[8]) + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -1031,34 +1041,38 @@ class Gtf(Gff): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: return False - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: - return False + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + int(hdr[4]) + except Exception: + return False + if hdr[5] != '.': try: - int(hdr[3]) - int(hdr[4]) + float(hdr[5]) except Exception: return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in data.valid_strand: - return False - if hdr[7] not in self.valid_gff_frame: - return False - - # Check attributes for gene_id (transcript_id is also mandatory - # but not for genes) - attributes = parse_gff_attributes(hdr[8]) - if 'gene_id' not in attributes: - return False - return True + if hdr[6] not in data.valid_strand: + return False + if hdr[7] not in self.valid_gff_frame: + return False + # Check attributes for gene_id (transcript_id is also mandatory + # but not for genes) + attributes = parse_gff_attributes(hdr[8]) + if 'gene_id' not in attributes: + return False + found_valid_lines = True + return found_valid_lines except Exception: return False diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 272042ece26..a63d5340bb9 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -235,24 +235,22 @@ def is_column_based(fname_or_file_prefix, sep='\t', skip=0): return False try: - headers = get_headers(fname_or_file_prefix, sep) + headers = get_headers(fname_or_file_prefix, sep, comment_designator='#')[skip:] except UnicodeDecodeError: return False count = 0 if not headers: return False - for hdr in headers[skip:]: - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) > 1: + for hdr in headers: + if hdr and hdr != ['']: + if count: + if len(hdr) != count: + return False + else: count = len(hdr) - break - if count < 2: - return False - for hdr in headers[skip:]: - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != count: - return False - return True + if count < 2: + return False + return count >= 2 def guess_ext(fname, sniff_order, is_binary=False): @@ -303,13 +301,13 @@ def guess_ext(fname, sniff_order, is_binary=False): >>> guess_ext(fname, sniff_order) 'gff3' >>> fname = get_test_fname('2.txt') - >>> guess_ext(fname, sniff_order) # 2.txt + >>> guess_ext(fname, sniff_order) 'txt' >>> fname = get_test_fname('2.tabular') >>> guess_ext(fname, sniff_order) 'tabular' >>> fname = get_test_fname('3.txt') - >>> guess_ext(fname, sniff_order) # 3.txt + >>> guess_ext(fname, sniff_order) 'txt' >>> fname = get_test_fname('test_tab1.tabular') >>> guess_ext(fname, sniff_order) @@ -451,6 +449,9 @@ def guess_ext(fname, sniff_order, is_binary=False): >>> fname = get_test_fname('1imzml') >>> guess_ext(fname, sniff_order) # This test case is ensuring doesn't throw exception, actual value could change if non-utf encoding handling improves. 'data' + >>> fname = get_test_fname('too_many_comments_gff3.tabular') + >>> guess_ext(fname, sniff_order) # It's a VCF but is sniffed as tabular because of the limit on the number of header lines we read + 'tabular' """ file_prefix = FilePrefix(fname) file_ext = run_sniffers_raw(file_prefix, sniff_order, is_binary) diff --git a/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular b/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular new file mode 100644 index 00000000000..e3fb5113dc6 --- /dev/null +++ b/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular @@ -0,0 +1,65 @@ +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +ctgA est match 5410 7503 . - . ID=EST:agt830.3;Target=agt830.3+1+595 +ctgA est HSP 7000 7503 . - . Parent=EST:agt830.3;Target=agt830.3+1+504 +ctgA est HSP 5410 5500 . ? . Parent=EST:agt830.3;Target=agt830.3+505+595 From ebf996f8bcd46c35807dd706ef8d35e1913b79d5 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 19 Feb 2021 04:56:37 +0000 Subject: [PATCH 10/10] Set Vcf metadata also for VcfGz files --- lib/galaxy/datatypes/tabular.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index a9513f9ccb0..ad23827bfc2 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -743,13 +743,12 @@ class BaseVcf(Tabular): def set_meta(self, dataset, **kwd): super().set_meta(dataset, **kwd) - source = open(dataset.file_name) - - # Skip comments. line = None - for line in source: - if not line.startswith('##'): - break + with compression_utils.get_fileobj(dataset.file_name) as fh: + # Skip comments. + for line in fh: + if not line.startswith('##'): + break if line and line.startswith('#'): # Found header line, get sample names. @@ -815,7 +814,7 @@ class VcfGz(BaseVcf, binary.Binary): return binascii.hexlify(last28) == b'1f8b08040000000000ff0600424302001b0003000000000000000000' def set_meta(self, dataset, **kwd): - super(BaseVcf, self).set_meta(dataset, **kwd) + super().set_meta(dataset, **kwd) """ Creates the index for the VCF file. """ # These metadata values are not accessible by users, always overwrite index_file = dataset.metadata.tabix_index