From 03e218df68fff72a175bf46c09fb67e5dd85d241 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 12 Jan 2022 12:09:42 +0100 Subject: [PATCH] Fix config schema packaging --- lib/galaxy/config/__init__.py | 9 +++++---- lib/galaxy/config/config_manage.py | 11 ++++++----- .../galaxy => config/schemas}/config_schema.yml | 0 .../galaxy => config/schemas}/job_config_schema.yml | 0 lib/galaxy/config/schemas/uwsgi_schema.yml | 1 + lib/galaxy/webapps/base/webapp.py | 4 ++-- lib/galaxy/webapps/galaxy/uwsgi_schema.yml | 1 - packages/app/MANIFEST.in | 2 +- packages/app/galaxy/config_schema.yml | 1 - packages/app/galaxy/job_config_schema.yml | 1 - packages/app/galaxy/uwsgi_schema.yml | 1 - packages/app/setup.py | 6 +++--- pyproject.toml | 1 + test/unit/app/jobs/test_job_configuration.py | 5 +++-- 14 files changed, 22 insertions(+), 21 deletions(-) rename lib/galaxy/{webapps/galaxy => config/schemas}/config_schema.yml (100%) rename lib/galaxy/{webapps/galaxy => config/schemas}/job_config_schema.yml (100%) create mode 120000 lib/galaxy/config/schemas/uwsgi_schema.yml delete mode 120000 lib/galaxy/webapps/galaxy/uwsgi_schema.yml delete mode 120000 packages/app/galaxy/config_schema.yml delete mode 120000 packages/app/galaxy/job_config_schema.yml delete mode 120000 packages/app/galaxy/uwsgi_schema.yml diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 097cbd907a2..b3253dcd39e 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -63,9 +63,9 @@ from galaxy.web_stack import get_stack_facts from ..version import VERSION_MAJOR, VERSION_MINOR try: - from importlib.resources import files + from importlib.resources import files # type: ignore[attr-defined] except ImportError: - from importlib_resources import files + from importlib_resources import files # type: ignore[no-redef] if TYPE_CHECKING: from galaxy.jobs import JobConfiguration @@ -77,7 +77,9 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) GALAXY_APP_NAME = 'galaxy' -GALAXY_CONFIG_SCHEMA_PATH = files('galaxy.webapps.galaxy') / 'config_schema.yml' +GALAXY_SCHEMAS_PATH = files('galaxy.config') / 'schemas' +GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'config_schema.yml' +UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'uwsgi_schema.yml' LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { 'disable_existing_loggers': False, 'version': 1, @@ -628,7 +630,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self._process_config(kwargs) def _load_schema(self): - # Schemas are symlinked to the root of the galaxy-app package return AppSchema(GALAXY_CONFIG_SCHEMA_PATH, GALAXY_APP_NAME) def _override_tempdir(self, kwargs): diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index 33bd8e8c6d6..526c0f158bd 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -22,7 +22,10 @@ if __name__ == '__main__': sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) -from galaxy.config import GALAXY_CONFIG_SCHEMA_PATH +from galaxy.config import ( + GALAXY_CONFIG_SCHEMA_PATH, + UWSGI_SCHEMA_PATH, +) from galaxy.config.schema import ( AppSchema, OPTION_DEFAULTS, @@ -48,7 +51,6 @@ UNHANDLED_FILTER_TYPE_MESSAGE = "Unhandled filter type encountered [%s] for sect NO_APP_MAIN_MESSAGE = "No app:main section found, using application defaults throughout." YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ", break_long_words=False, break_on_hyphens=False) RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent=" ", subsequent_indent=" ", break_long_words=False, break_on_hyphens=False) -UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml" UWSGI_OPTIONS = dict([ ('http', { @@ -346,7 +348,7 @@ GALAXY_APP = App( "8080", ["galaxy.web.buildapp:app_factory"], # TODO: Galaxy could call factory a few different things and they'd all be fine. "config/galaxy.yml", - GALAXY_CONFIG_SCHEMA_PATH, + str(GALAXY_CONFIG_SCHEMA_PATH), 'galaxy.webapps.galaxy.buildapp:uwsgi_app()', ) SHED_APP = App( @@ -465,9 +467,8 @@ def _build_uwsgi_schema(args, app_desc): "desc": "uwsgi definition, see https://uwsgi-docs.readthedocs.io/en/latest/Options.html", "mapping": options } - path = os.path.join(args.galaxy_root, UWSGI_SCHEMA_PATH) contents = ordered_dump(schema) - _write_to_file(args, contents, path) + _write_to_file(args, contents, UWSGI_SCHEMA_PATH) def _find_config(args, app_desc): diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml similarity index 100% rename from lib/galaxy/webapps/galaxy/config_schema.yml rename to lib/galaxy/config/schemas/config_schema.yml diff --git a/lib/galaxy/webapps/galaxy/job_config_schema.yml b/lib/galaxy/config/schemas/job_config_schema.yml similarity index 100% rename from lib/galaxy/webapps/galaxy/job_config_schema.yml rename to lib/galaxy/config/schemas/job_config_schema.yml diff --git a/lib/galaxy/config/schemas/uwsgi_schema.yml b/lib/galaxy/config/schemas/uwsgi_schema.yml new file mode 120000 index 00000000000..d9366b0e83f --- /dev/null +++ b/lib/galaxy/config/schemas/uwsgi_schema.yml @@ -0,0 +1 @@ +../../../../lib/galaxy/webapps/uwsgi_schema.yml \ No newline at end of file diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 374cf0afb29..f0b5d5b66a2 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -42,9 +42,9 @@ from galaxy.web.framework import ( from galaxy.web_stack import get_app_kwds try: - from importlib.resources import files + from importlib.resources import files # type: ignore[attr-defined] except ImportError: - from importlib_resources import files + from importlib_resources import files # type: ignore[no-redef] log = logging.getLogger(__name__) diff --git a/lib/galaxy/webapps/galaxy/uwsgi_schema.yml b/lib/galaxy/webapps/galaxy/uwsgi_schema.yml deleted file mode 120000 index 7d0729284c9..00000000000 --- a/lib/galaxy/webapps/galaxy/uwsgi_schema.yml +++ /dev/null @@ -1 +0,0 @@ -../uwsgi_schema.yml \ No newline at end of file diff --git a/packages/app/MANIFEST.in b/packages/app/MANIFEST.in index 438b5168639..2f0c85e5c48 100644 --- a/packages/app/MANIFEST.in +++ b/packages/app/MANIFEST.in @@ -1,5 +1,5 @@ include *.rst *.txt LICENSE -include galaxy/*.yml +include galaxy/config/*.yml include galaxy/config/sample/*.sample* include galaxy/jobs/runners/util/job_script/*.sh include galaxy/tools/*tsv diff --git a/packages/app/galaxy/config_schema.yml b/packages/app/galaxy/config_schema.yml deleted file mode 120000 index 6c9a0d37301..00000000000 --- a/packages/app/galaxy/config_schema.yml +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/webapps/galaxy/config_schema.yml \ No newline at end of file diff --git a/packages/app/galaxy/job_config_schema.yml b/packages/app/galaxy/job_config_schema.yml deleted file mode 120000 index 075efbd9219..00000000000 --- a/packages/app/galaxy/job_config_schema.yml +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/webapps/galaxy/job_config_schema.yml \ No newline at end of file diff --git a/packages/app/galaxy/uwsgi_schema.yml b/packages/app/galaxy/uwsgi_schema.yml deleted file mode 120000 index 7a34c111575..00000000000 --- a/packages/app/galaxy/uwsgi_schema.yml +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/webapps/galaxy/uwsgi_schema.yml \ No newline at end of file diff --git a/packages/app/setup.py b/packages/app/setup.py index 072b7f8063b..02c1bf87530 100644 --- a/packages/app/setup.py +++ b/packages/app/setup.py @@ -101,9 +101,9 @@ ENTRY_POINTS = ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. 'galaxy': [ - 'config_schema.yml', - 'job_config_schema.yml', - 'uwsgi_schema.yml', + 'config/schemas/config_schema.yml', + 'config/schemas/job_config_schema.yml', + 'config/schemas/uwsgi_schema.yml', 'config/sample/*', ], 'tool_shed': [ diff --git a/pyproject.toml b/pyproject.toml index 690beb6b130..c758328aeb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ future = "*" galaxy_sequence_utils = "*" gxformat2 = "*" h5py = "*" +importlib_resources = "*" isa-rwval = "*" kombu = "*" lagom = "*" diff --git a/test/unit/app/jobs/test_job_configuration.py b/test/unit/app/jobs/test_job_configuration.py index 0e36c8c41c7..a4dde99ca53 100644 --- a/test/unit/app/jobs/test_job_configuration.py +++ b/test/unit/app/jobs/test_job_configuration.py @@ -7,6 +7,7 @@ from unittest import mock from pykwalify.core import Core +from galaxy.config import GALAXY_SCHEMAS_PATH from galaxy.job_metrics import JobMetrics from galaxy.jobs import JobConfiguration from galaxy.util import galaxy_directory, galaxy_samples_directory @@ -422,7 +423,7 @@ class AdvancedJobConfYamlParserTestCase(AdvancedJobConfXmlParserTestCase): def test_yaml_advanced_validation(): - schema = os.path.join(galaxy_directory(), "lib", "galaxy", "webapps", "galaxy", "job_config_schema.yml") + schema = GALAXY_SCHEMAS_PATH / 'job_config_schema.yml' integration_tests_dir = os.path.join(galaxy_directory(), "test", "integration") valid_files = [ ADVANCED_JOB_CONF_YAML, @@ -435,6 +436,6 @@ def test_yaml_advanced_validation(): for valid_file in valid_files: c = Core( source_file=valid_file, - schema_files=[schema], + schema_files=[str(schema)], ) c.validate()