From d69a6fcd0e119238b4812abd1bc6bfa517195142 Mon Sep 17 00:00:00 2001 From: rikeshi Date: Sat, 13 Feb 2021 11:50:17 +0100 Subject: [PATCH 1/8] remove kwds arg from call to get_stored_accessible_workflow fix 'key' causing TypeError on GET request --- lib/galaxy/webapps/galaxy/api/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 214a5fe1bb9..f09e5a46ba5 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -244,7 +244,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn Lists all versions of this workflow. """ - stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id, **kwds) + stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id) return [{'version': i, 'update_time': str(w.update_time), 'steps': len(w.steps)} for i, w in enumerate(reversed(stored_workflow.workflows))] @expose_api From 9cc01de07a6478113dc8ca6ff379bce42893c55e Mon Sep 17 00:00:00 2001 From: rikeshi Date: Mon, 15 Feb 2021 17:22:30 +0000 Subject: [PATCH 2/8] Apply suggestions from code review Co-authored-by: Simon Bray <32272674+simonbray@users.noreply.github.com> --- lib/galaxy/webapps/galaxy/api/workflows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index f09e5a46ba5..2b9ef427d2d 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -244,7 +244,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn Lists all versions of this workflow. """ - stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id) + instance = util.string_as_bool(kwds.get("instance", "false")) + stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id, by_stored_id=not instance) return [{'version': i, 'update_time': str(w.update_time), 'steps': len(w.steps)} for i, w in enumerate(reversed(stored_workflow.workflows))] @expose_api From e2076fbb9dd7c79913a4c696d9c1eed5ec031add Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Feb 2021 18:08:31 +0100 Subject: [PATCH 3/8] fix genration of config files --- lib/galaxy/tools/parameters/wrapped_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index b65063c7262..a0574981292 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -97,7 +97,7 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"): if input.multiple and packaging.version.parse(str(profile)) >= packaging.version.parse('20.05'): json_value = [_ for _ in _cast_if_not_none(value_wrapper.value, list)] else: - json_value = _cast_if_not_none(value_wrapper, str) + json_value = _cast_if_not_none(value_wrapper.value, str) elif input_type == "data_column": # value is a SelectToolParameterWrapper() if input.multiple: From bd5e8f326cc63efe3a2711836c10a57d88b9ca9d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Feb 2021 19:22:29 +0100 Subject: [PATCH 4/8] fix profile logic for configfile generation some follow up fixes to https://github.com/galaxyproject/galaxy/pull/9279/files https://github.com/galaxyproject/galaxy/pull/9776/files - wrong profile for the profile test tool (it used 20.01, but should have used 20.05) - the non-profile test tool used wrong assumptions. now fixed and also tested against 19.09 - in the profile version `value_wrapper.value` needs to be casted for `multiple==False` otherwise we gat `"None"` (which I do not understand 100%) --- lib/galaxy/tools/parameters/wrapped_json.py | 9 ++++++--- test/functional/tools/inputs_as_json.xml | 6 +++--- test/functional/tools/inputs_as_json_profile.xml | 8 ++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index a0574981292..9a3393ffc21 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -94,10 +94,13 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"): elif input_type == "boolean": json_value = _cast_if_not_none(value_wrapper, bool) elif input_type == "select": - if input.multiple and packaging.version.parse(str(profile)) >= packaging.version.parse('20.05'): - json_value = [_ for _ in _cast_if_not_none(value_wrapper.value, list)] + if packaging.version.parse(str(profile)) < packaging.version.parse('20.05'): + json_value = _cast_if_not_none(value_wrapper, str) else: - json_value = _cast_if_not_none(value_wrapper.value, str) + if input.multiple: + json_value = [str(_) for _ in _cast_if_not_none(value_wrapper.value, list)] + else: + json_value = _cast_if_not_none(value_wrapper.value, str) elif input_type == "data_column": # value is a SelectToolParameterWrapper() if input.multiple: diff --git a/test/functional/tools/inputs_as_json.xml b/test/functional/tools/inputs_as_json.xml index 465aba5eda2..1aff602e796 100644 --- a/test/functional/tools/inputs_as_json.xml +++ b/test/functional/tools/inputs_as_json.xml @@ -27,8 +27,8 @@ if test_case == "1": assert_equals(as_dict["inttest"], 12456) assert_equals(as_dict["floattest"], 6.789) assert_equals(as_dict["radio_select"], "a_radio") - assert_equals(as_dict["optional_select"], None) - assert_equals(as_dict["optional_multiple_select"], []) + assert_equals(as_dict["optional_select"], "None") + assert_equals(as_dict["optional_multiple_select"], "None") assert_equals(as_dict["repeat"][0]["r"], "000000") assert_equals(as_dict["repeat"][1]["r"], "FFFFFF") assert_equals(as_dict["cond"]["more_text"], "fdefault") @@ -43,7 +43,7 @@ elif test_case == "2": assert_equals(as_dict["floattest"], 1.0) assert_equals(as_dict["radio_select"], "a_radio") assert_equals(as_dict["optional_select"], "a") - assert_equals(as_dict["optional_multiple_select"], ['a', 'b']) + assert_equals(as_dict["optional_multiple_select"], 'a,b') assert_equals(as_dict["repeat"][0]["r"], "000000") assert_equals(as_dict["cond"]["cond_test"], "second") assert_equals(as_dict["cond"]["more_text"], "sdefault") diff --git a/test/functional/tools/inputs_as_json_profile.xml b/test/functional/tools/inputs_as_json_profile.xml index 9fc04164878..162a6fa7345 100644 --- a/test/functional/tools/inputs_as_json_profile.xml +++ b/test/functional/tools/inputs_as_json_profile.xml @@ -1,7 +1,7 @@ - + Date: Fri, 15 Jan 2021 19:06:56 +0000 Subject: [PATCH 5/8] Sync social-auth-core requirement in data package v4.0.2 dropped support for Python 3.5 --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- packages/data/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 762cfc3a441..27fcd08ea2c 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -82,7 +82,7 @@ pyparsing = "*" "Fabric3" = "*" paramiko = "*" cloudbridge = "*" -social_auth_core = {version = "==3.3.0", extras = ['openidconnect']} +social-auth-core = {version = "==3.3.0", extras = ['openidconnect']} cloudauthz = "==0.6.0" gxformat2 = "*" refgenconf = ">=0.7.0" diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 98424dcbef5..8753dc5cddd 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -9,7 +9,7 @@ parsley numpy<=1.16 pycryptodome pysam -social_auth_core +social-auth-core[openidconnect]==3.3.0 SQLAlchemy sqlalchemy-migrate sqlalchemy-utils From fc1d74d1d3a7d8c9d5a07dd71a14ab8caaec1474 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 8 Feb 2021 11:19:41 +0100 Subject: [PATCH 6/8] Deploy postgresql and rabbitmq in kubernetes Setting up minikube stops the postgres and rabbitmq containers that are set up by the workflow. These are setup with custom docker networks, so it isn't trivial to restart them. Instead we just deploy them to minikube. --- .ci/minikube-test-setup/deployment.yaml | 32 +++++++++ .ci/minikube-test-setup/start_services.sh | 12 ++++ .github/workflows/integration.yaml | 85 +++++++++++++---------- 3 files changed, 92 insertions(+), 37 deletions(-) create mode 100644 .ci/minikube-test-setup/deployment.yaml create mode 100644 .ci/minikube-test-setup/start_services.sh diff --git a/.ci/minikube-test-setup/deployment.yaml b/.ci/minikube-test-setup/deployment.yaml new file mode 100644 index 00000000000..d7afe110c83 --- /dev/null +++ b/.ci/minikube-test-setup/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: testing + name: testing +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: test + template: + metadata: + labels: + app.kubernetes.io/name: test + spec: + containers: + - image: postgres:12 + name: postgres + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: postgres + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + value: postgres + - image: rabbitmq + name: rabbitmq + ports: + - containerPort: 5672 diff --git a/.ci/minikube-test-setup/start_services.sh b/.ci/minikube-test-setup/start_services.sh new file mode 100644 index 00000000000..1e860dadcb7 --- /dev/null +++ b/.ci/minikube-test-setup/start_services.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -ex + +SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") +kubectl apply -f "$SCRIPTDIR/deployment.yaml" +kubectl expose deployment testing --type=LoadBalancer --name=testing-service + +CLUSTER_IP=$(kubectl get service testing-service -o jsonpath='{.spec.clusterIP}') +GALAXY_TEST_DBURI="postgresql://postgres:postgres@${CLUSTER_IP}:5432/galaxy?client_encoding=utf-8" +GALAXY_TEST_AMQP_URL="amqp://${CLUSTER_IP}:5672)//" +export GALAXY_TEST_DBURI +export GALAXY_TEST_AMQP_URL diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 3bd0aaf4d69..c48e5346388 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -6,14 +6,15 @@ env: jobs: test: name: Test - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: [3.7] - subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)'] + python-version: ['3.7'] + subset: ['kubernetes'] services: postgres: - image: postgres:11 + image: postgres:13 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -25,36 +26,46 @@ jobs: ports: - 5672:5672 steps: - - name: Prune unused docker image, volumes and containers - run: docker system prune -a -f - - name: Clean dotnet folder for space - if: matrix.subset == 'kubernetes' - run: rm -Rf /usr/share/dotnet - - name: Setup Minikube - if: matrix.subset == 'kubernetes' - id: minikube - uses: CodingNagger/minikube-setup-action@v1.0.3 - with: - minikube-version: "1.9.0-0_amd64" - - name: Launch Minikube - if: matrix.subset == 'kubernetes' - run: eval ${{ steps.minikube.outputs.launcher }} - - name: Check pods - if: matrix.subset == 'kubernetes' - run: | - kubectl get pods - - uses: actions/checkout@v2 - with: - path: 'galaxy root' - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip dir - uses: actions/cache@v1 - id: pip-cache - with: - path: ~/.cache/pip - key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} - - name: Run tests - run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' - working-directory: 'galaxy root' + - name: Prune unused docker image, volumes and containers + run: docker system prune -a -f + - name: Clean dotnet folder for space + if: matrix.subset == 'kubernetes' + run: rm -Rf /usr/share/dotnet + - name: Setup Minikube + if: matrix.subset == 'kubernetes' + id: minikube + uses: CodingNagger/minikube-setup-action@v1.0.3 + with: + minikube-version: "1.9.0-0_amd64" + - name: Launch Minikube + if: matrix.subset == 'kubernetes' + run: eval ${{ steps.minikube.outputs.launcher }} + - name: Check pods + if: matrix.subset == 'kubernetes' + run: | + kubectl get pods + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip dir + uses: actions/cache@v1 + id: pip-cache + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Run tests + if: matrix.subset != 'kubernetes' + run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + working-directory: 'galaxy root' + - name: Run tests + if: matrix.subset == 'kubernetes' + run: 'source .ci/minikube-test-setup/start_services.sh && ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + working-directory: 'galaxy root' + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: Integration test results (${{ matrix.python-version }}, ${{ matrix.subset }}) + path: 'galaxy root/run_integration_tests.html' From a8adab100ccf38cfc2a14de3c798479f5531213e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 8 Feb 2021 13:08:45 +0100 Subject: [PATCH 7/8] Restore all integration subsets --- .github/workflows/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index c48e5346388..410af8045a9 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: python-version: ['3.7'] - subset: ['kubernetes'] + subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)'] services: postgres: image: postgres:13 From db1618fc6a3c7454ab2d1ed8af8ea6d38aeb66e7 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Mon, 8 Feb 2021 14:01:28 +0100 Subject: [PATCH 8/8] Use multiline yaml Co-authored-by: Nicola Soranzo --- .github/workflows/integration.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 410af8045a9..5cba3dba72b 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -62,7 +62,9 @@ jobs: working-directory: 'galaxy root' - name: Run tests if: matrix.subset == 'kubernetes' - run: 'source .ci/minikube-test-setup/start_services.sh && ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' + run: | + . .ci/minikube-test-setup/start_services.sh + ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}" working-directory: 'galaxy root' - uses: actions/upload-artifact@v2 if: failure()