Merge pull request #13147 from mvdbeek/mako_fix

lib/galaxy/config/__init__.py

@@ -62,6 +62,12 @@ from galaxy.web.formatting import expand_pretty_datetime_format
 from galaxy.web_stack import get_stack_facts
 from ..version import VERSION_MAJOR, VERSION_MINOR
 
+try:
+    from importlib.resources import files  # type: ignore[attr-defined]
+except ImportError:
+    # Python < 3.9
+    from importlib_resources import files  # type: ignore[no-redef]
+
 if TYPE_CHECKING:
     from galaxy.jobs import JobConfiguration
     from galaxy.tool_util.deps.containers import ContainerFinder
@@ -72,7 +78,9 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 
 GALAXY_APP_NAME = 'galaxy'
-GALAXY_CONFIG_SCHEMA_PATH = 'lib/galaxy/webapps/galaxy/config_schema.yml'
+GALAXY_SCHEMAS_PATH = files('galaxy.config') / 'schemas'
+GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'config_schema.yml'
+UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'uwsgi_schema.yml'
 LOGGING_CONFIG_DEFAULT: Dict[str, Any] = {
     'disable_existing_loggers': False,
     'version': 1,
@@ -623,14 +631,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
         self._process_config(kwargs)
 
     def _load_schema(self):
-        # Schemas are symlinked to the root of the galaxy-app package
-        config_schema_path = os.path.join(os.path.dirname(__file__), os.pardir, 'config_schema.yml')
-        if os.path.exists(GALAXY_CONFIG_SCHEMA_PATH):
-            config_schema_path = GALAXY_CONFIG_SCHEMA_PATH
-        elif not os.path.exists(config_schema_path):
-            # Not a package, but cwd is not galaxy_root
-            config_schema_path = os.path.join(self.root, GALAXY_CONFIG_SCHEMA_PATH)
-        return AppSchema(config_schema_path, GALAXY_APP_NAME)
+        return AppSchema(GALAXY_CONFIG_SCHEMA_PATH, GALAXY_APP_NAME)
 
     def _override_tempdir(self, kwargs):
         if string_as_bool(kwargs.get("override_tempdir", "True")):
@@ -740,7 +741,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
             for ip in kwargs.get("fetch_url_allowlist", "").split(',')
             if len(ip.strip()) > 0
         ]
-        self.template_path = self._in_root_dir(kwargs.get("template_path", "templates"))
         self.job_queue_cleanup_interval = int(kwargs.get("job_queue_cleanup_interval", "5"))
         self.cluster_files_directory = self._in_root_dir(self.cluster_files_directory)
 

lib/galaxy/config/config_manage.py

@@ -22,7 +22,10 @@ if __name__ == '__main__':
     sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)))
 
 
-from galaxy.config import GALAXY_CONFIG_SCHEMA_PATH
+from galaxy.config import (
+    GALAXY_CONFIG_SCHEMA_PATH,
+    UWSGI_SCHEMA_PATH,
+)
 from galaxy.config.schema import (
     AppSchema,
     OPTION_DEFAULTS,
@@ -48,7 +51,6 @@ UNHANDLED_FILTER_TYPE_MESSAGE = "Unhandled filter type encountered [%s] for sect
 NO_APP_MAIN_MESSAGE = "No app:main section found, using application defaults throughout."
 YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ", break_long_words=False, break_on_hyphens=False)
 RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent="    ", subsequent_indent="    ", break_long_words=False, break_on_hyphens=False)
-UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml"
 
 UWSGI_OPTIONS = dict([
     ('http', {
@@ -346,7 +348,7 @@ GALAXY_APP = App(
     "8080",
     ["galaxy.web.buildapp:app_factory"],  # TODO: Galaxy could call factory a few different things and they'd all be fine.
     "config/galaxy.yml",
-    GALAXY_CONFIG_SCHEMA_PATH,
+    str(GALAXY_CONFIG_SCHEMA_PATH),
     'galaxy.webapps.galaxy.buildapp:uwsgi_app()',
 )
 SHED_APP = App(
@@ -465,9 +467,8 @@ def _build_uwsgi_schema(args, app_desc):
         "desc": "uwsgi definition, see https://uwsgi-docs.readthedocs.io/en/latest/Options.html",
         "mapping": options
     }
-    path = os.path.join(args.galaxy_root, UWSGI_SCHEMA_PATH)
     contents = ordered_dump(schema)
-    _write_to_file(args, contents, path)
+    _write_to_file(args, contents, UWSGI_SCHEMA_PATH)
 
 
 def _find_config(args, app_desc):

lib/galaxy/webapps/base/webapp.py

@@ -41,6 +41,12 @@ from galaxy.web.framework import (
 )
 from galaxy.web_stack import get_app_kwds
 
+try:
+    from importlib.resources import files  # type: ignore[attr-defined]
+except ImportError:
+    # Python < 3.9
+    from importlib_resources import files  # type: ignore[no-redef]
+
 log = logging.getLogger(__name__)
 
 
@@ -79,8 +85,8 @@ class WebApplication(base.WebApplication):
     injection_aware: bool = False
 
     def __init__(self, galaxy_app, session_cookie='galaxysession', name=None):
+        super().__init__()
         self.name = name
-        base.WebApplication.__init__(self)
         galaxy_app.is_webapp = True
         self.set_transaction_factory(lambda e: self.transaction_chooser(e, galaxy_app, session_cookie))
         # Mako support
@@ -90,16 +96,13 @@ class WebApplication(base.WebApplication):
 
     def create_mako_template_lookup(self, galaxy_app, name):
         paths = []
-        # FIXME: should be os.path.join (galaxy_root, 'templates')?
-        if galaxy_app.config.template_path == './templates':
-            template_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates'))
-        else:
-            template_path = galaxy_app.config.template_path
+        base_package = 'tool_shed.webapp' if galaxy_app.name == 'tool_shed' else 'galaxy.webapps.base'  # reports has templates in galaxy package
+        base_template_path = files(base_package) / 'templates'
         # First look in webapp specific directory
         if name is not None:
-            paths.append(os.path.join(template_path, 'webapps', name))
+            paths.append(base_template_path / 'webapps' / name)
         # Then look in root directory
-        paths.append(template_path)
+        paths.append(base_template_path)
         # Create TemplateLookup with a small cache
         return mako.lookup.TemplateLookup(directories=paths,
                                           module_directory=galaxy_app.config.template_cache_path,

lib/galaxy/webapps/galaxy/uwsgi_schema.yml

@@ -1 +0,0 @@
-../uwsgi_schema.yml

lib/galaxy/webapps/galaxy/workers.py

@@ -5,5 +5,5 @@ try:
     import uvloop  # noqa: F401
     from uvicorn.workers import UvicornWorker as Worker
 except ImportError:
-    log.warning("uvtools not available, falling back to pure python worker")
+    log.warning("uvloop not available, falling back to pure python worker")
     from uvicorn.workers import UvicornH11Worker as Worker  # noqa: F401

lib/galaxy/webapps/reports/config.py

@@ -33,7 +33,6 @@ class Configuration:
         self.id_secret = kwargs.get("id_secret", "USING THE DEFAULT IS NOT SECURE!")
         self.use_remote_user = string_as_bool(kwargs.get("use_remote_user", "False"))
         self.require_login = string_as_bool(kwargs.get("require_login", "False"))
-        self.template_path = resolve_path(kwargs.get("template_path", "templates"), self.root)
         self.template_cache_path = resolve_path(kwargs.get("template_cache_path", "database/compiled_templates/reports"), self.root)
         self.allow_user_creation = string_as_bool(kwargs.get("allow_user_creation", "True"))
         self.allow_user_deletion = string_as_bool(kwargs.get("allow_user_deletion", "False"))
@@ -66,7 +65,7 @@ class Configuration:
 
     def check(self):
         # Check that required directories exist
-        for path in self.root, self.template_path:
+        for path in (self.root, ):
             if not os.path.isdir(path):
                 raise ConfigurationError(f"Directory does not exist: {path}")
 

lib/galaxy/webapps/reports/uwsgi_schema.yml

@@ -1 +1 @@
-../uwsgi_schema.yml
+../../config/schemas/uwsgi_schema.yml

lib/galaxy_test/driver/driver_util.py

@@ -249,7 +249,6 @@ def setup_galaxy_config(
         master_api_key=master_api_key,
         running_functional_tests=True,
         template_cache_path=template_cache_path,
-        template_path='templates',
         tool_config_file=tool_config_file,
         tool_data_table_config_path=tool_data_table_config_path,
         tool_parse_help=False,

lib/tool_shed/test/functional_tests.py

@@ -89,7 +89,6 @@ class ToolShedTestDriver(driver_util.TestDriver):
                       shed_tool_data_table_config=shed_tool_data_table_conf_file,
                       smtp_server='smtp.dummy.string.tld',
                       email_from='functional@localhost',
-                      template_path='templates',
                       tool_parse_help=False,
                       use_heartbeat=False)
         kwargs.update(toolshed_database_conf)

lib/tool_shed/webapp/config.py

@@ -18,13 +18,16 @@ from galaxy.util import string_as_bool
 from galaxy.version import VERSION, VERSION_MAJOR, VERSION_MINOR
 from galaxy.web.formatting import expand_pretty_datetime_format
 
+try:
+    from importlib.resources import files  # type: ignore[attr-defined]
+except ImportError:
+    # Python < 3.9
+    from importlib_resources import files  # type: ignore[no-redef]
+
 log = logging.getLogger(__name__)
 
-ts_webapp_path = os.path.abspath(os.path.dirname(__file__))
-templates_path = os.path.join(ts_webapp_path, 'templates')
-
 TOOLSHED_APP_NAME = 'tool_shed'
-TOOLSHED_CONFIG_SCHEMA_PATH = 'lib/tool_shed/webapp/config_schema.yml'
+TOOLSHED_CONFIG_SCHEMA_PATH = files('tool_shed.webapp') / 'config_schema.yml'
 
 
 class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
@@ -48,7 +51,6 @@ class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
         paths_to_check = [
             self.file_path,
             self.hgweb_config_dir,
-            self.template_path,
             self.tool_data_path,
             self.template_cache_path,
             os.path.join(self.tool_data_path, 'shared', 'jars'),
@@ -95,7 +97,6 @@ class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
         self.registration_warning_message = kwargs.get('registration_warning_message')
         self.email_domain_blocklist_content = None
         self.email_domain_allowlist_content = None
-        self.template_path = templates_path
         self.template_cache_path = self._in_root_dir(kwargs.get('template_cache_path', 'database/compiled_templates/community'))
         self.error_email_to = kwargs.get('error_email_to')
         self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format)

lib/tool_shed/webapp/uwsgi_schema.yml

@@ -1 +1 @@
-../../galaxy/webapps/uwsgi_schema.yml
+../../galaxy/config/schemas/uwsgi_schema.yml

packages/app/MANIFEST.in

@@ -1,5 +1,5 @@
 include *.rst *.txt LICENSE
-include galaxy/*.yml
+include galaxy/config/schemas/*.yml
 include galaxy/config/sample/*.sample*
 include galaxy/jobs/runners/util/job_script/*.sh
 include galaxy/tools/*tsv

packages/app/galaxy/config_schema.yml

@@ -1 +0,0 @@
-../../../lib/galaxy/webapps/galaxy/config_schema.yml

packages/app/galaxy/job_config_schema.yml

@@ -1 +0,0 @@
-../../../lib/galaxy/webapps/galaxy/job_config_schema.yml

packages/app/galaxy/uwsgi_schema.yml

@@ -1 +0,0 @@
-../../../lib/galaxy/webapps/galaxy/uwsgi_schema.yml

packages/app/setup.py

@@ -101,9 +101,7 @@ ENTRY_POINTS = '''
 PACKAGE_DATA = {
     # Be sure to update MANIFEST.in for source dist.
     'galaxy': [
-        'config_schema.yml',
-        'job_config_schema.yml',
-        'uwsgi_schema.yml',
+        'config/schemas/*.yml',
         'config/sample/*',
     ],
     'tool_shed': [

packages/webapps/requirements.txt

@@ -2,6 +2,7 @@ galaxy-app
 Cheetah3
 fastapi>=0.68.2,!=0.69.0,!=0.70.0,!=0.70.1
 fastapi-utils
+importlib_resources
 Mako
 pydantic
 python-multipart  # required to support form parsing in FastAPI/Starlette

pyproject.toml

@@ -40,6 +40,7 @@ future = "*"
 galaxy_sequence_utils = "*"
 gxformat2 = "*"
 h5py = "*"
+importlib_resources = "*"
 isa-rwval = "*"
 kombu = "*"
 lagom = "*"

scripts/release-diff.py

@@ -142,7 +142,13 @@ def load_at_time(path, revision=None):
 
 
 def main(old_revision, new_revision=None):
-    files_to_diff = glob.glob("config/*.yml.sample") + glob.glob('lib/galaxy/webapps/galaxy/*schema.yml')
+    globs = (
+        "config/*.yml.sample",
+        "lib/galaxy/config/schemas/*schema.yml",
+        "lib/galaxy/webapps/reports/config_schema.yml",
+        "lib/tool_shed/webapp/config_schema.yml",
+    )
+    files_to_diff = [f for g in globs for f in glob.glob(g)]
     added = {}
     removed = {}
     changed = {}

test/unit/app/jobs/test_job_configuration.py

@@ -7,6 +7,7 @@ from unittest import mock
 
 from pykwalify.core import Core
 
+from galaxy.config import GALAXY_SCHEMAS_PATH
 from galaxy.job_metrics import JobMetrics
 from galaxy.jobs import JobConfiguration
 from galaxy.util import galaxy_directory, galaxy_samples_directory
@@ -422,7 +423,7 @@ class AdvancedJobConfYamlParserTestCase(AdvancedJobConfXmlParserTestCase):
 
 
 def test_yaml_advanced_validation():
-    schema = os.path.join(galaxy_directory(), "lib", "galaxy", "webapps", "galaxy", "job_config_schema.yml")
+    schema = GALAXY_SCHEMAS_PATH / 'job_config_schema.yml'
     integration_tests_dir = os.path.join(galaxy_directory(), "test", "integration")
     valid_files = [
         ADVANCED_JOB_CONF_YAML,
@@ -435,6 +436,6 @@ def test_yaml_advanced_validation():
     for valid_file in valid_files:
         c = Core(
             source_file=valid_file,
-            schema_files=[schema],
+            schema_files=[str(schema)],
         )
         c.validate()

test/unit/webapps/test_routes.py

@@ -22,8 +22,8 @@ class TestWebapp(WebApplication):
 
 
 def test_galaxy_routes():
-    test_config = Bunch(template_path="/tmp", template_cache_path="/tmp")
-    app = Bunch(config=test_config, security=object(), trace_logger=None)
+    test_config = Bunch(template_cache_path="/tmp")
+    app = Bunch(config=test_config, security=object(), trace_logger=None, name="galaxy")
     test_webapp = TestWebapp(app)
 
     galaxy_buildapp.populate_api_routes(test_webapp, app)