From be10a32aa97215a1e4cbff9737b305abc473fd0d Mon Sep 17 00:00:00 2001 From: dglaetzer <35349535+dglaetzer@users.noreply.github.com> Date: Fri, 5 Jul 2019 11:10:07 +0200 Subject: [PATCH 01/13] run tests for tool_util --- packages/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test.sh b/packages/test.sh index aacfab67a3e..2079be976ca 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -25,7 +25,7 @@ PACKAGE_DIRS=( ) # containers has no tests, tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. -RUN_TESTS=(1 1 1 0 0 0 0 1) +RUN_TESTS=(1 1 1 0 1 0 0 1) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]} From 9cf0924578394ce85460a705b035226c5adb5c99 Mon Sep 17 00:00:00 2001 From: dglaetzer <35349535+dglaetzer@users.noreply.github.com> Date: Fri, 5 Jul 2019 14:08:09 +0200 Subject: [PATCH 02/13] Update test.sh --- packages/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test.sh b/packages/test.sh index 2079be976ca..34024fbe642 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -25,7 +25,7 @@ PACKAGE_DIRS=( ) # containers has no tests, tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. -RUN_TESTS=(1 1 1 0 1 0 0 1) +RUN_TESTS=(0 0 0 0 0 1 0) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]} From 54d2ca01f879b19222edf538205b48e2fc55fa3b Mon Sep 17 00:00:00 2001 From: damian Date: Sat, 6 Jul 2019 13:58:52 +0200 Subject: [PATCH 03/13] moved common_util.py, encoding_util.py and xml_util.py from tool_shed/util to galaxy/util/tool_shed (for data package) --- lib/galaxy/datatypes/converters/maf_to_fasta_converter.py | 2 +- .../datatypes/converters/maf_to_interval_converter.py | 2 +- lib/galaxy/datatypes/test/temp2.txt | 2 ++ lib/galaxy/{tools => datatypes}/util/maf_utilities.py | 0 lib/galaxy/model/migrate/check.py | 1 - lib/galaxy/model/tool_shed_install/__init__.py | 2 +- lib/galaxy/util/tool_shed/__init__.py | 0 .../util => galaxy/util/tool_shed}/common_util.py | 6 ++++-- .../util => galaxy/util/tool_shed}/encoding_util.py | 2 -- lib/{tool_shed/util => galaxy/util/tool_shed}/xml_util.py | 0 packages/data/requirements.txt | 3 +++ packages/data/tests/unittest_utils/__init__.py | 2 +- packages/data/tests/unittest_utils/tempfilecache.py | 2 +- packages/data/tests/unittest_utils/utility.py | 2 +- packages/util/requirements.txt | 2 ++ 15 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 lib/galaxy/datatypes/test/temp2.txt rename lib/galaxy/{tools => datatypes}/util/maf_utilities.py (100%) create mode 100644 lib/galaxy/util/tool_shed/__init__.py rename lib/{tool_shed/util => galaxy/util/tool_shed}/common_util.py (99%) rename lib/{tool_shed/util => galaxy/util/tool_shed}/encoding_util.py (95%) rename lib/{tool_shed/util => galaxy/util/tool_shed}/xml_util.py (100%) diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py index 98e6af16c01..1b893f1ab5d 100644 --- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py @@ -6,7 +6,7 @@ import sys import bx.align.maf -from galaxy.tools.util import maf_utilities +from galaxy.datatypes.util import maf_utilities assert sys.version_info[:2] >= (2, 6) diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py index 51049ed54c6..a19ab25cabb 100644 --- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py @@ -6,7 +6,7 @@ import sys import bx.align.maf -from galaxy.tools.util import maf_utilities +from galaxy.datatypes.util import maf_utilities assert sys.version_info[:2] >= (2, 6) diff --git a/lib/galaxy/datatypes/test/temp2.txt b/lib/galaxy/datatypes/test/temp2.txt new file mode 100644 index 00000000000..2c3f803432c --- /dev/null +++ b/lib/galaxy/datatypes/test/temp2.txt @@ -0,0 +1,2 @@ +1 2 +3 4 diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/datatypes/util/maf_utilities.py similarity index 100% rename from lib/galaxy/tools/util/maf_utilities.py rename to lib/galaxy/datatypes/util/maf_utilities.py diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index 3fe5f987446..e90576434e6 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -1,6 +1,5 @@ import logging import os.path -import sys from migrate.versioning import repository, schema from sqlalchemy import ( diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index 4aef91b43b7..668943720a5 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -4,7 +4,7 @@ import os from galaxy.util import asbool from galaxy.util.bunch import Bunch from galaxy.util.dictifiable import Dictifiable -from tool_shed.util import common_util +from galaxy.util.tool_shed import common_util log = logging.getLogger(__name__) diff --git a/lib/galaxy/util/tool_shed/__init__.py b/lib/galaxy/util/tool_shed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/tool_shed/util/common_util.py b/lib/galaxy/util/tool_shed/common_util.py similarity index 99% rename from lib/tool_shed/util/common_util.py rename to lib/galaxy/util/tool_shed/common_util.py index 8f5686197a2..84bb1182489 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/galaxy/util/tool_shed/common_util.py @@ -1,13 +1,15 @@ import json import logging import os +import sys + +from routes import url_for from six.moves.urllib.parse import urljoin from galaxy import util from galaxy.util.odict import odict -from galaxy.web import url_for -from tool_shed.util import encoding_util, xml_util +from galaxy.util.tool_shed import encoding_util, xml_util log = logging.getLogger(__name__) diff --git a/lib/tool_shed/util/encoding_util.py b/lib/galaxy/util/tool_shed/encoding_util.py similarity index 95% rename from lib/tool_shed/util/encoding_util.py rename to lib/galaxy/util/tool_shed/encoding_util.py index f5bb2ff6441..57c1a11c8a7 100644 --- a/lib/tool_shed/util/encoding_util.py +++ b/lib/galaxy/util/tool_shed/encoding_util.py @@ -1,6 +1,5 @@ import binascii import json -import logging from galaxy.util import ( smart_str, @@ -8,7 +7,6 @@ from galaxy.util import ( ) from galaxy.util.hash_util import hmac_new -log = logging.getLogger(__name__) encoding_sep = '__esep__' encoding_sep2 = '__esepii__' diff --git a/lib/tool_shed/util/xml_util.py b/lib/galaxy/util/tool_shed/xml_util.py similarity index 100% rename from lib/tool_shed/util/xml_util.py rename to lib/galaxy/util/tool_shed/xml_util.py diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index f2b5c80f06e..c053000ed2e 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -1,13 +1,16 @@ galaxy-objectstore +galaxy-sequence-utils==1.1.3 galaxy-util bdbag bx-python h5py isa-rwval numpy +parsley pycryptodome pysam social_auth_core SQLAlchemy==1.2 sqlalchemy-migrate +sqlalchemy-utils WebOb diff --git a/packages/data/tests/unittest_utils/__init__.py b/packages/data/tests/unittest_utils/__init__.py index 2115ad35783..290bb9ee29b 120000 --- a/packages/data/tests/unittest_utils/__init__.py +++ b/packages/data/tests/unittest_utils/__init__.py @@ -1 +1 @@ -../../test/unit/unittest_utils/__init__.py \ No newline at end of file +../../../../test/unit/unittest_utils/__init__.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/tempfilecache.py b/packages/data/tests/unittest_utils/tempfilecache.py index 93fb83f1df7..588f6eec7d8 120000 --- a/packages/data/tests/unittest_utils/tempfilecache.py +++ b/packages/data/tests/unittest_utils/tempfilecache.py @@ -1 +1 @@ -../../test/unit/unittest_utils/tempfilecache.py \ No newline at end of file +../../../../test/unit/unittest_utils/tempfilecache.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/utility.py b/packages/data/tests/unittest_utils/utility.py index 54dd4ca7a3b..3c69a8fd67e 120000 --- a/packages/data/tests/unittest_utils/utility.py +++ b/packages/data/tests/unittest_utils/utility.py @@ -1 +1 @@ -../../test/unit/unittest_utils/utility.py \ No newline at end of file +../../../../test/unit/unittest_utils/utility.py \ No newline at end of file diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index e15e5db251e..01367630095 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -2,7 +2,9 @@ bleach boltons bz2file; python_version < '3.3' docutils +logging markupsafe packaging pyyaml +routes six>=1.9.0 From 64b94a2b862d4915855c3a5a734cf0dae899c562 Mon Sep 17 00:00:00 2001 From: damian Date: Sat, 6 Jul 2019 14:12:21 +0200 Subject: [PATCH 04/13] symlinked tool shed utils back to their original directory --- packages/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test.sh b/packages/test.sh index 34024fbe642..64444f86b6d 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -25,7 +25,7 @@ PACKAGE_DIRS=( ) # containers has no tests, tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. -RUN_TESTS=(0 0 0 0 0 1 0) +RUN_TESTS=(1 1 1 0 0 1 0) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]} From ac3f74b483c24bf52c4b36d1a87889ae34b0d586 Mon Sep 17 00:00:00 2001 From: dglaetzer <35349535+dglaetzer@users.noreply.github.com> Date: Sat, 6 Jul 2019 14:24:17 +0200 Subject: [PATCH 05/13] accidentialy removed it before --- lib/galaxy/model/migrate/check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index e90576434e6..3fe5f987446 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -1,5 +1,6 @@ import logging import os.path +import sys from migrate.versioning import repository, schema from sqlalchemy import ( From 3289e2341052f1a39cd5d4851309e1d3484fb6b1 Mon Sep 17 00:00:00 2001 From: dglaetzer <35349535+dglaetzer@users.noreply.github.com> Date: Sat, 6 Jul 2019 14:27:21 +0200 Subject: [PATCH 06/13] added it accidentialy before --- lib/galaxy/util/tool_shed/common_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/util/tool_shed/common_util.py b/lib/galaxy/util/tool_shed/common_util.py index 84bb1182489..e171c16237c 100644 --- a/lib/galaxy/util/tool_shed/common_util.py +++ b/lib/galaxy/util/tool_shed/common_util.py @@ -1,7 +1,6 @@ import json import logging import os -import sys from routes import url_for From e2e6cbffb899cca28dada322b1bec583bf126999 Mon Sep 17 00:00:00 2001 From: dglaetzer <35349535+dglaetzer@users.noreply.github.com> Date: Sat, 6 Jul 2019 15:29:39 +0200 Subject: [PATCH 07/13] Update common_util.py --- lib/galaxy/util/tool_shed/common_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/util/tool_shed/common_util.py b/lib/galaxy/util/tool_shed/common_util.py index e171c16237c..1faa08ac6b4 100644 --- a/lib/galaxy/util/tool_shed/common_util.py +++ b/lib/galaxy/util/tool_shed/common_util.py @@ -3,7 +3,6 @@ import logging import os from routes import url_for - from six.moves.urllib.parse import urljoin from galaxy import util From 1b6be605b89ca0300f2737742ec2805b520fe501 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sat, 6 Jul 2019 17:40:13 +0200 Subject: [PATCH 08/13] Fix imports --- packages/util/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index 01367630095..3aaba044dcb 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -1,8 +1,9 @@ bleach boltons bz2file; python_version < '3.3' +cheetah3 docutils -logging +future markupsafe packaging pyyaml From 560565b4e6f442b751bac16844166a693b8e8e0f Mon Sep 17 00:00:00 2001 From: dglaetzer Date: Sun, 7 Jul 2019 10:56:17 +0200 Subject: [PATCH 09/13] link maf_utilities back to it's original directory, fix requirements --- packages/data/requirements.txt | 2 +- packages/util/requirements.txt | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 874087c5c5a..40d2a9baed0 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -1,6 +1,6 @@ galaxy-objectstore galaxy-sequence-utils==1.1.3 -galaxy-util +galaxy-util[template] bdbag bx-python h5py diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index 3aaba044dcb..aad3e2dc97a 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -1,9 +1,7 @@ bleach boltons bz2file; python_version < '3.3' -cheetah3 docutils -future markupsafe packaging pyyaml From b51c20ddfebf369f99cd3f34d1afd65e82688c42 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 7 Jul 2019 13:50:39 +0200 Subject: [PATCH 10/13] Fix typo in run_tests.sh help text --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 1236e8742cc..08322c750a5 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -11,7 +11,7 @@ cat < Date: Sun, 7 Jul 2019 18:55:51 +0200 Subject: [PATCH 11/13] linked maf_utilities.py back to it's original place (now, finally) --- lib/galaxy/tools/util/maf_utilities.py | 1 + 1 file changed, 1 insertion(+) create mode 120000 lib/galaxy/tools/util/maf_utilities.py diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py new file mode 120000 index 00000000000..07bafe2d1a5 --- /dev/null +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -0,0 +1 @@ +../../datatypes/util/maf_utilities.py \ No newline at end of file From a9f6c0cd1af4f31fea2ac4e55aaa91983826d738 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 7 Jul 2019 19:05:59 +0200 Subject: [PATCH 12/13] Refactor config_watchers out of webapps so can be in galaxy-app package. --- lib/galaxy/app.py | 2 +- lib/galaxy/{webapps/galaxy => }/config_watchers.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/galaxy/{webapps/galaxy => }/config_watchers.py (100%) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 03d56cac84b..f704bd72bb0 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -11,6 +11,7 @@ import galaxy.queues import galaxy.quota import galaxy.security from galaxy import config, job_metrics, jobs +from galaxy.config_watchers import ConfigWatchers from galaxy.containers import build_container_interfaces from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.folders import FolderManager @@ -40,7 +41,6 @@ from galaxy.visualization.plugins.registry import VisualizationsRegistry from galaxy.web import url_for from galaxy.web.proxy import ProxyManager from galaxy.web_stack import application_stack_instance -from galaxy.webapps.galaxy.config_watchers import ConfigWatchers from galaxy.webhooks import WebhooksRegistry from tool_shed.galaxy_install import ( installed_repository_manager, diff --git a/lib/galaxy/webapps/galaxy/config_watchers.py b/lib/galaxy/config_watchers.py similarity index 100% rename from lib/galaxy/webapps/galaxy/config_watchers.py rename to lib/galaxy/config_watchers.py From 1815a6547a2f796430c68764a59b2bae7aa2ad79 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 7 Jul 2019 19:09:34 +0200 Subject: [PATCH 13/13] Bring top-level galaxy Python files into galaxy-app. --- packages/app/galaxy/app.py | 1 + packages/app/galaxy/config.py | 1 + packages/app/galaxy/config_watchers.py | 1 + packages/app/galaxy/main.py | 1 + packages/app/galaxy/queue_worker.py | 1 + packages/app/galaxy/queues.py | 1 + packages/app/galaxy/version.py | 1 + test/unit/tools/test_toolbox.py | 2 +- 8 files changed, 8 insertions(+), 1 deletion(-) create mode 120000 packages/app/galaxy/app.py create mode 120000 packages/app/galaxy/config.py create mode 120000 packages/app/galaxy/config_watchers.py create mode 120000 packages/app/galaxy/main.py create mode 120000 packages/app/galaxy/queue_worker.py create mode 120000 packages/app/galaxy/queues.py create mode 120000 packages/app/galaxy/version.py diff --git a/packages/app/galaxy/app.py b/packages/app/galaxy/app.py new file mode 120000 index 00000000000..61ec0a3dd4d --- /dev/null +++ b/packages/app/galaxy/app.py @@ -0,0 +1 @@ +../../../lib/galaxy/app.py \ No newline at end of file diff --git a/packages/app/galaxy/config.py b/packages/app/galaxy/config.py new file mode 120000 index 00000000000..7e61acb3d31 --- /dev/null +++ b/packages/app/galaxy/config.py @@ -0,0 +1 @@ +../../../lib/galaxy/config.py \ No newline at end of file diff --git a/packages/app/galaxy/config_watchers.py b/packages/app/galaxy/config_watchers.py new file mode 120000 index 00000000000..ffd3e6bd109 --- /dev/null +++ b/packages/app/galaxy/config_watchers.py @@ -0,0 +1 @@ +../../../lib/galaxy/config_watchers.py \ No newline at end of file diff --git a/packages/app/galaxy/main.py b/packages/app/galaxy/main.py new file mode 120000 index 00000000000..6b458fd3ff8 --- /dev/null +++ b/packages/app/galaxy/main.py @@ -0,0 +1 @@ +../../../lib/galaxy/main.py \ No newline at end of file diff --git a/packages/app/galaxy/queue_worker.py b/packages/app/galaxy/queue_worker.py new file mode 120000 index 00000000000..7fcb68fbea9 --- /dev/null +++ b/packages/app/galaxy/queue_worker.py @@ -0,0 +1 @@ +../../../lib/galaxy/queue_worker.py \ No newline at end of file diff --git a/packages/app/galaxy/queues.py b/packages/app/galaxy/queues.py new file mode 120000 index 00000000000..a5afab45290 --- /dev/null +++ b/packages/app/galaxy/queues.py @@ -0,0 +1 @@ +../../../lib/galaxy/queues.py \ No newline at end of file diff --git a/packages/app/galaxy/version.py b/packages/app/galaxy/version.py new file mode 120000 index 00000000000..42d972d9994 --- /dev/null +++ b/packages/app/galaxy/version.py @@ -0,0 +1 @@ +../../../lib/galaxy/version.py \ No newline at end of file diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index f16b565e693..e592f8e43d2 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -9,11 +9,11 @@ import routes from six import string_types from galaxy import model +from galaxy.config_watchers import ConfigWatchers from galaxy.model import tool_shed_install from galaxy.model.tool_shed_install import mapping from galaxy.tools import ToolBox from galaxy.tools.cache import ToolCache -from galaxy.webapps.galaxy.config_watchers import ConfigWatchers from .test_tool_loader import ( SIMPLE_MACRO, SIMPLE_TOOL_WITH_MACRO