diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index b7607fbe45e..59e42daecf8 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -62,6 +62,12 @@ from galaxy.web.formatting import expand_pretty_datetime_format from galaxy.web_stack import get_stack_facts from ..version import VERSION_MAJOR, VERSION_MINOR +try: + from importlib.resources import files # type: ignore[attr-defined] +except ImportError: + # Python < 3.9 + from importlib_resources import files # type: ignore[no-redef] + if TYPE_CHECKING: from galaxy.jobs import JobConfiguration from galaxy.tool_util.deps.containers import ContainerFinder @@ -72,7 +78,9 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) GALAXY_APP_NAME = 'galaxy' -GALAXY_CONFIG_SCHEMA_PATH = 'lib/galaxy/webapps/galaxy/config_schema.yml' +GALAXY_SCHEMAS_PATH = files('galaxy.config') / 'schemas' +GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'config_schema.yml' +UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'uwsgi_schema.yml' LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { 'disable_existing_loggers': False, 'version': 1, @@ -623,14 +631,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self._process_config(kwargs) def _load_schema(self): - # Schemas are symlinked to the root of the galaxy-app package - config_schema_path = os.path.join(os.path.dirname(__file__), os.pardir, 'config_schema.yml') - if os.path.exists(GALAXY_CONFIG_SCHEMA_PATH): - config_schema_path = GALAXY_CONFIG_SCHEMA_PATH - elif not os.path.exists(config_schema_path): - # Not a package, but cwd is not galaxy_root - config_schema_path = os.path.join(self.root, GALAXY_CONFIG_SCHEMA_PATH) - return AppSchema(config_schema_path, GALAXY_APP_NAME) + return AppSchema(GALAXY_CONFIG_SCHEMA_PATH, GALAXY_APP_NAME) def _override_tempdir(self, kwargs): if string_as_bool(kwargs.get("override_tempdir", "True")): @@ -740,7 +741,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): for ip in kwargs.get("fetch_url_allowlist", "").split(',') if len(ip.strip()) > 0 ] - self.template_path = self._in_root_dir(kwargs.get("template_path", "templates")) self.job_queue_cleanup_interval = int(kwargs.get("job_queue_cleanup_interval", "5")) self.cluster_files_directory = self._in_root_dir(self.cluster_files_directory) diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index 33bd8e8c6d6..526c0f158bd 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -22,7 +22,10 @@ if __name__ == '__main__': sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) -from galaxy.config import GALAXY_CONFIG_SCHEMA_PATH +from galaxy.config import ( + GALAXY_CONFIG_SCHEMA_PATH, + UWSGI_SCHEMA_PATH, +) from galaxy.config.schema import ( AppSchema, OPTION_DEFAULTS, @@ -48,7 +51,6 @@ UNHANDLED_FILTER_TYPE_MESSAGE = "Unhandled filter type encountered [%s] for sect NO_APP_MAIN_MESSAGE = "No app:main section found, using application defaults throughout." YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ", break_long_words=False, break_on_hyphens=False) RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent=" ", subsequent_indent=" ", break_long_words=False, break_on_hyphens=False) -UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml" UWSGI_OPTIONS = dict([ ('http', { @@ -346,7 +348,7 @@ GALAXY_APP = App( "8080", ["galaxy.web.buildapp:app_factory"], # TODO: Galaxy could call factory a few different things and they'd all be fine. "config/galaxy.yml", - GALAXY_CONFIG_SCHEMA_PATH, + str(GALAXY_CONFIG_SCHEMA_PATH), 'galaxy.webapps.galaxy.buildapp:uwsgi_app()', ) SHED_APP = App( @@ -465,9 +467,8 @@ def _build_uwsgi_schema(args, app_desc): "desc": "uwsgi definition, see https://uwsgi-docs.readthedocs.io/en/latest/Options.html", "mapping": options } - path = os.path.join(args.galaxy_root, UWSGI_SCHEMA_PATH) contents = ordered_dump(schema) - _write_to_file(args, contents, path) + _write_to_file(args, contents, UWSGI_SCHEMA_PATH) def _find_config(args, app_desc): diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml similarity index 100% rename from lib/galaxy/webapps/galaxy/config_schema.yml rename to lib/galaxy/config/schemas/config_schema.yml diff --git a/lib/galaxy/webapps/galaxy/job_config_schema.yml b/lib/galaxy/config/schemas/job_config_schema.yml similarity index 100% rename from lib/galaxy/webapps/galaxy/job_config_schema.yml rename to lib/galaxy/config/schemas/job_config_schema.yml diff --git a/lib/galaxy/webapps/uwsgi_schema.yml b/lib/galaxy/config/schemas/uwsgi_schema.yml similarity index 100% rename from lib/galaxy/webapps/uwsgi_schema.yml rename to lib/galaxy/config/schemas/uwsgi_schema.yml diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 902b1974294..9a99007f1af 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -41,6 +41,12 @@ from galaxy.web.framework import ( ) from galaxy.web_stack import get_app_kwds +try: + from importlib.resources import files # type: ignore[attr-defined] +except ImportError: + # Python < 3.9 + from importlib_resources import files # type: ignore[no-redef] + log = logging.getLogger(__name__) @@ -79,8 +85,8 @@ class WebApplication(base.WebApplication): injection_aware: bool = False def __init__(self, galaxy_app, session_cookie='galaxysession', name=None): + super().__init__() self.name = name - base.WebApplication.__init__(self) galaxy_app.is_webapp = True self.set_transaction_factory(lambda e: self.transaction_chooser(e, galaxy_app, session_cookie)) # Mako support @@ -90,16 +96,13 @@ class WebApplication(base.WebApplication): def create_mako_template_lookup(self, galaxy_app, name): paths = [] - # FIXME: should be os.path.join (galaxy_root, 'templates')? - if galaxy_app.config.template_path == './templates': - template_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')) - else: - template_path = galaxy_app.config.template_path + base_package = 'tool_shed.webapp' if galaxy_app.name == 'tool_shed' else 'galaxy.webapps.base' # reports has templates in galaxy package + base_template_path = files(base_package) / 'templates' # First look in webapp specific directory if name is not None: - paths.append(os.path.join(template_path, 'webapps', name)) + paths.append(base_template_path / 'webapps' / name) # Then look in root directory - paths.append(template_path) + paths.append(base_template_path) # Create TemplateLookup with a small cache return mako.lookup.TemplateLookup(directories=paths, module_directory=galaxy_app.config.template_cache_path, diff --git a/lib/galaxy/webapps/galaxy/uwsgi_schema.yml b/lib/galaxy/webapps/galaxy/uwsgi_schema.yml deleted file mode 120000 index 7d0729284c9..00000000000 --- a/lib/galaxy/webapps/galaxy/uwsgi_schema.yml +++ /dev/null @@ -1 +0,0 @@ -../uwsgi_schema.yml \ No newline at end of file diff --git a/lib/galaxy/webapps/galaxy/workers.py b/lib/galaxy/webapps/galaxy/workers.py index b153b232820..2f1300fe923 100644 --- a/lib/galaxy/webapps/galaxy/workers.py +++ b/lib/galaxy/webapps/galaxy/workers.py @@ -5,5 +5,5 @@ try: import uvloop # noqa: F401 from uvicorn.workers import UvicornWorker as Worker except ImportError: - log.warning("uvtools not available, falling back to pure python worker") + log.warning("uvloop not available, falling back to pure python worker") from uvicorn.workers import UvicornH11Worker as Worker # noqa: F401 diff --git a/lib/galaxy/webapps/reports/config.py b/lib/galaxy/webapps/reports/config.py index 5f653fbe35d..1f65b90833b 100644 --- a/lib/galaxy/webapps/reports/config.py +++ b/lib/galaxy/webapps/reports/config.py @@ -33,7 +33,6 @@ class Configuration: self.id_secret = kwargs.get("id_secret", "USING THE DEFAULT IS NOT SECURE!") self.use_remote_user = string_as_bool(kwargs.get("use_remote_user", "False")) self.require_login = string_as_bool(kwargs.get("require_login", "False")) - self.template_path = resolve_path(kwargs.get("template_path", "templates"), self.root) self.template_cache_path = resolve_path(kwargs.get("template_cache_path", "database/compiled_templates/reports"), self.root) self.allow_user_creation = string_as_bool(kwargs.get("allow_user_creation", "True")) self.allow_user_deletion = string_as_bool(kwargs.get("allow_user_deletion", "False")) @@ -66,7 +65,7 @@ class Configuration: def check(self): # Check that required directories exist - for path in self.root, self.template_path: + for path in (self.root, ): if not os.path.isdir(path): raise ConfigurationError(f"Directory does not exist: {path}") diff --git a/lib/galaxy/webapps/reports/uwsgi_schema.yml b/lib/galaxy/webapps/reports/uwsgi_schema.yml index 7d0729284c9..321ce78cbe3 120000 --- a/lib/galaxy/webapps/reports/uwsgi_schema.yml +++ b/lib/galaxy/webapps/reports/uwsgi_schema.yml @@ -1 +1 @@ -../uwsgi_schema.yml \ No newline at end of file +../../config/schemas/uwsgi_schema.yml \ No newline at end of file diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index aff24b1abb7..07e8ad2a81b 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -249,7 +249,6 @@ def setup_galaxy_config( master_api_key=master_api_key, running_functional_tests=True, template_cache_path=template_cache_path, - template_path='templates', tool_config_file=tool_config_file, tool_data_table_config_path=tool_data_table_config_path, tool_parse_help=False, diff --git a/lib/tool_shed/test/functional_tests.py b/lib/tool_shed/test/functional_tests.py index 2e35eaf3488..5bc08abc79a 100644 --- a/lib/tool_shed/test/functional_tests.py +++ b/lib/tool_shed/test/functional_tests.py @@ -89,7 +89,6 @@ class ToolShedTestDriver(driver_util.TestDriver): shed_tool_data_table_config=shed_tool_data_table_conf_file, smtp_server='smtp.dummy.string.tld', email_from='functional@localhost', - template_path='templates', tool_parse_help=False, use_heartbeat=False) kwargs.update(toolshed_database_conf) diff --git a/lib/tool_shed/webapp/config.py b/lib/tool_shed/webapp/config.py index cdbb987e126..094186b431b 100644 --- a/lib/tool_shed/webapp/config.py +++ b/lib/tool_shed/webapp/config.py @@ -18,13 +18,16 @@ from galaxy.util import string_as_bool from galaxy.version import VERSION, VERSION_MAJOR, VERSION_MINOR from galaxy.web.formatting import expand_pretty_datetime_format +try: + from importlib.resources import files # type: ignore[attr-defined] +except ImportError: + # Python < 3.9 + from importlib_resources import files # type: ignore[no-redef] + log = logging.getLogger(__name__) -ts_webapp_path = os.path.abspath(os.path.dirname(__file__)) -templates_path = os.path.join(ts_webapp_path, 'templates') - TOOLSHED_APP_NAME = 'tool_shed' -TOOLSHED_CONFIG_SCHEMA_PATH = 'lib/tool_shed/webapp/config_schema.yml' +TOOLSHED_CONFIG_SCHEMA_PATH = files('tool_shed.webapp') / 'config_schema.yml' class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): @@ -48,7 +51,6 @@ class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): paths_to_check = [ self.file_path, self.hgweb_config_dir, - self.template_path, self.tool_data_path, self.template_cache_path, os.path.join(self.tool_data_path, 'shared', 'jars'), @@ -95,7 +97,6 @@ class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.registration_warning_message = kwargs.get('registration_warning_message') self.email_domain_blocklist_content = None self.email_domain_allowlist_content = None - self.template_path = templates_path self.template_cache_path = self._in_root_dir(kwargs.get('template_cache_path', 'database/compiled_templates/community')) self.error_email_to = kwargs.get('error_email_to') self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format) diff --git a/lib/tool_shed/webapp/uwsgi_schema.yml b/lib/tool_shed/webapp/uwsgi_schema.yml index de49dde4140..3710028ca76 120000 --- a/lib/tool_shed/webapp/uwsgi_schema.yml +++ b/lib/tool_shed/webapp/uwsgi_schema.yml @@ -1 +1 @@ -../../galaxy/webapps/uwsgi_schema.yml \ No newline at end of file +../../galaxy/config/schemas/uwsgi_schema.yml \ No newline at end of file diff --git a/packages/app/MANIFEST.in b/packages/app/MANIFEST.in index 438b5168639..a4acb5c99e4 100644 --- a/packages/app/MANIFEST.in +++ b/packages/app/MANIFEST.in @@ -1,5 +1,5 @@ include *.rst *.txt LICENSE -include galaxy/*.yml +include galaxy/config/schemas/*.yml include galaxy/config/sample/*.sample* include galaxy/jobs/runners/util/job_script/*.sh include galaxy/tools/*tsv diff --git a/packages/app/galaxy/config_schema.yml b/packages/app/galaxy/config_schema.yml deleted file mode 120000 index 6c9a0d37301..00000000000 --- a/packages/app/galaxy/config_schema.yml +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/webapps/galaxy/config_schema.yml \ No newline at end of file diff --git a/packages/app/galaxy/job_config_schema.yml b/packages/app/galaxy/job_config_schema.yml deleted file mode 120000 index 075efbd9219..00000000000 --- a/packages/app/galaxy/job_config_schema.yml +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/webapps/galaxy/job_config_schema.yml \ No newline at end of file diff --git a/packages/app/galaxy/uwsgi_schema.yml b/packages/app/galaxy/uwsgi_schema.yml deleted file mode 120000 index 7a34c111575..00000000000 --- a/packages/app/galaxy/uwsgi_schema.yml +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/webapps/galaxy/uwsgi_schema.yml \ No newline at end of file diff --git a/packages/app/setup.py b/packages/app/setup.py index 072b7f8063b..4812c204bfe 100644 --- a/packages/app/setup.py +++ b/packages/app/setup.py @@ -101,9 +101,7 @@ ENTRY_POINTS = ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. 'galaxy': [ - 'config_schema.yml', - 'job_config_schema.yml', - 'uwsgi_schema.yml', + 'config/schemas/*.yml', 'config/sample/*', ], 'tool_shed': [ diff --git a/packages/webapps/requirements.txt b/packages/webapps/requirements.txt index a932b6c2948..7a195cd8db1 100644 --- a/packages/webapps/requirements.txt +++ b/packages/webapps/requirements.txt @@ -2,6 +2,7 @@ galaxy-app Cheetah3 fastapi>=0.68.2,!=0.69.0,!=0.70.0,!=0.70.1 fastapi-utils +importlib_resources Mako pydantic python-multipart # required to support form parsing in FastAPI/Starlette diff --git a/pyproject.toml b/pyproject.toml index 690beb6b130..c758328aeb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ future = "*" galaxy_sequence_utils = "*" gxformat2 = "*" h5py = "*" +importlib_resources = "*" isa-rwval = "*" kombu = "*" lagom = "*" diff --git a/scripts/release-diff.py b/scripts/release-diff.py index f2e84d8ad94..240faa90dc7 100644 --- a/scripts/release-diff.py +++ b/scripts/release-diff.py @@ -142,7 +142,13 @@ def load_at_time(path, revision=None): def main(old_revision, new_revision=None): - files_to_diff = glob.glob("config/*.yml.sample") + glob.glob('lib/galaxy/webapps/galaxy/*schema.yml') + globs = ( + "config/*.yml.sample", + "lib/galaxy/config/schemas/*schema.yml", + "lib/galaxy/webapps/reports/config_schema.yml", + "lib/tool_shed/webapp/config_schema.yml", + ) + files_to_diff = [f for g in globs for f in glob.glob(g)] added = {} removed = {} changed = {} diff --git a/test/unit/app/jobs/test_job_configuration.py b/test/unit/app/jobs/test_job_configuration.py index 0e36c8c41c7..a4dde99ca53 100644 --- a/test/unit/app/jobs/test_job_configuration.py +++ b/test/unit/app/jobs/test_job_configuration.py @@ -7,6 +7,7 @@ from unittest import mock from pykwalify.core import Core +from galaxy.config import GALAXY_SCHEMAS_PATH from galaxy.job_metrics import JobMetrics from galaxy.jobs import JobConfiguration from galaxy.util import galaxy_directory, galaxy_samples_directory @@ -422,7 +423,7 @@ class AdvancedJobConfYamlParserTestCase(AdvancedJobConfXmlParserTestCase): def test_yaml_advanced_validation(): - schema = os.path.join(galaxy_directory(), "lib", "galaxy", "webapps", "galaxy", "job_config_schema.yml") + schema = GALAXY_SCHEMAS_PATH / 'job_config_schema.yml' integration_tests_dir = os.path.join(galaxy_directory(), "test", "integration") valid_files = [ ADVANCED_JOB_CONF_YAML, @@ -435,6 +436,6 @@ def test_yaml_advanced_validation(): for valid_file in valid_files: c = Core( source_file=valid_file, - schema_files=[schema], + schema_files=[str(schema)], ) c.validate() diff --git a/test/unit/webapps/test_routes.py b/test/unit/webapps/test_routes.py index 2c4191c2d2d..7fe1485395a 100644 --- a/test/unit/webapps/test_routes.py +++ b/test/unit/webapps/test_routes.py @@ -22,8 +22,8 @@ class TestWebapp(WebApplication): def test_galaxy_routes(): - test_config = Bunch(template_path="/tmp", template_cache_path="/tmp") - app = Bunch(config=test_config, security=object(), trace_logger=None) + test_config = Bunch(template_cache_path="/tmp") + app = Bunch(config=test_config, security=object(), trace_logger=None, name="galaxy") test_webapp = TestWebapp(app) galaxy_buildapp.populate_api_routes(test_webapp, app)