Merge pull request #13396 from mvdbeek/a2wsgi

[22.01] Use a2wsgi to serve WSGI app

lib/galaxy/dependencies/dev-requirements.txt

@@ -1,5 +1,6 @@
 --extra-index-url https://wheels.galaxyproject.org/simple
 
+a2wsgi==1.4.0; python_version >= "3.6" and python_version < "4.0"
 adal==1.2.7
 aiofiles==0.8.0; python_version >= "3.6" and python_version < "4.0"
 alabaster==0.7.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"

lib/galaxy/dependencies/pinned-requirements.txt

@@ -1,5 +1,6 @@
 --extra-index-url https://wheels.galaxyproject.org/simple
 
+a2wsgi==1.4.0; python_version >= "3.6" and python_version < "4.0"
 adal==1.2.7
 aiofiles==0.8.0; python_version >= "3.6" and python_version < "4.0"
 amqp==5.0.9; python_version >= "3.7"

lib/galaxy/managers/base.py

@@ -576,8 +576,6 @@ class ModelSerializer(HasAModelManager[T]):
         keys_to_serialize = [ 'id', 'name', 'attr1', 'attr2', ... ]
         item_dict = MySerializer.serialize( my_item, keys_to_serialize )
     """
-    #: 'service' to use for getting urls - use class var to allow overriding when testing
-    url_for = staticmethod(gx_url_for)
     default_view: Optional[str]
     views: Dict[str, List[str]]
 
@@ -604,6 +602,12 @@ class ModelSerializer(HasAModelManager[T]):
         self.views = {}
         self.default_view = None
 
+    @staticmethod
+    def url_for(*args, context=None, **kwargs):
+        trans = context and context.get('trans')
+        url_for = trans and trans.url_builder or gx_url_for
+        return url_for(*args, **kwargs)
+
     def add_serializers(self):
         """
         Register a map of attribute keys -> serializing functions that will serialize

lib/galaxy/managers/datasets.py

@@ -527,10 +527,11 @@ class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer[T],
             if getattr(dataset_assoc.metadata, meta_type, None):
                 meta_files.append(
                     dict(file_type=meta_type,
-                         download_url=self.url_for('history_contents_metadata_file',
+                         download_url=self.url_for('get_metadata_file',
                                                    history_id=self.app.security.encode_id(dataset_assoc.history_id),
                                                    history_content_id=self.app.security.encode_id(dataset_assoc.id),
-                                                   metadata_file=meta_type)))
+                                                   query_params={'metadata_file': meta_type},
+                                                   context=context)))
         return meta_files
 
     def serialize_metadata(self, item, key, excluded=None, **context):

lib/galaxy/managers/hdas.py

@@ -427,13 +427,15 @@ class HDASerializer(  # datasets._UnflattenedMetadataDatasetAssociationSerialize
             #   see also: https://sentry.galaxyproject.org/galaxy/galaxy-main/group/20769/events/9352883/
             'url': lambda item, key, **context: self.url_for('history_content',
                                                              history_id=self.app.security.encode_id(item.history_id),
-                                                             id=self.app.security.encode_id(item.id)),
+                                                             id=self.app.security.encode_id(item.id),
+                                                             context=context),
             'urls': self.serialize_urls,
 
             # TODO: backwards compat: need to go away
             'download_url': lambda item, key, **context: self.url_for('history_contents_display',
                                                                       history_id=self.app.security.encode_id(item.history.id),
-                                                                      history_content_id=self.app.security.encode_id(item.id)),
+                                                                      history_content_id=self.app.security.encode_id(item.id),
+                                                                      context=context),
             'parent_id': self.serialize_id,
             # TODO: to DatasetAssociationSerializer
             'accessible': lambda item, key, user=None, **c: self.manager.is_accessible(item, user, **c),

lib/galaxy/webapps/base/webapp.py

@@ -192,7 +192,8 @@ def config_allows_origin(origin_raw, config):
 
 
 def url_builder(*args, **kwargs) -> str:
-    """Wrapper around the uWSGI version of the function for reversing URLs."""
+    """Wrapper around the WSGI version of the function for reversing URLs."""
+    kwargs.update(kwargs.pop('query_params', {}))
     return url_for(*args, **kwargs)
 
 

lib/galaxy/webapps/galaxy/api/__init__.py

@@ -10,6 +10,7 @@ from typing import (
     Type,
     TypeVar,
 )
+from urllib.parse import urlencode
 
 from fastapi import (
     Cookie,
@@ -149,12 +150,20 @@ class UrlBuilder:
 
     def __call__(self, name: str, **path_params):
         qualified = path_params.pop("qualified", False)
+        # starlette does not support query parameters in url_path_for: https://github.com/encode/starlette/issues/560
+        query_params = path_params.pop('query_params', None)
         try:
             if qualified:
-                return self.request.url_for(name, **path_params)
-            return self.request.app.url_path_for(name, **path_params)
+                url = self.request.url_for(name, **path_params)
+            else:
+                url = self.request.app.url_path_for(name, **path_params)
+            if query_params:
+                url = f"{url}?{urlencode(query_params)}"
+            return url
         except NoMatchFound:
             # Fallback to legacy url_for
+            if query_params:
+                path_params.update(query_params)
             return web.url_for(name, **path_params)
 
 

lib/galaxy/webapps/galaxy/buildapp.py

@@ -297,7 +297,7 @@ def populate_api_routes(webapp, app):
                           controller='datasets',
                           action='show_inheritance_chain',
                           conditions=dict(method=["GET"]))
-    webapp.mapper.connect("history_contents_metadata_file",
+    webapp.mapper.connect("get_metadata_file",
                           "/api/histories/{history_id}/contents/{history_content_id}/metadata_file",
                           controller="datasets",
                           action="get_metadata_file",

lib/galaxy/webapps/galaxy/fast_app.py

@@ -1,8 +1,8 @@
 from pathlib import Path
 from typing import cast
 
+from a2wsgi import WSGIMiddleware
 from fastapi import FastAPI, Request
-from fastapi.middleware.wsgi import WSGIMiddleware
 from starlette.middleware.cors import CORSMiddleware
 from starlette.responses import (
     FileResponse,

lib/galaxy/webapps/reports/fast_app.py

@@ -1,5 +1,5 @@
+from a2wsgi import WSGIMiddleware
 from fastapi import FastAPI
-from fastapi.middleware.wsgi import WSGIMiddleware
 
 from galaxy.webapps.base.api import (
     add_exception_handler,

lib/galaxy_test/api/test_histories.py

@@ -1,4 +1,5 @@
 import time
+from uuid import uuid4
 
 from requests import (
     put
@@ -245,7 +246,7 @@ class ImportExportTests(BaseHistories):
         self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
 
     def test_import_export(self):
-        history_name = "for_export_default"
+        history_name = f"for_export_default_{uuid4()}"
         history_id = self.dataset_populator.setup_history_for_export_testing(history_name)
         imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=2)
 
@@ -272,7 +273,7 @@ class ImportExportTests(BaseHistories):
         self._import_history_and_wait(import_data, "API Test History", wait_on_history_length=2)
 
     def test_import_export_include_deleted(self):
-        history_name = "for_export_include_deleted"
+        history_name = f"for_export_include_deleted_{uuid4()}"
         history_id = self.dataset_populator.new_history(name=history_name)
         self.dataset_populator.new_dataset(history_id, content="1 2 3")
         deleted_hda = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True)
@@ -300,7 +301,7 @@ class ImportExportTests(BaseHistories):
 
     @skip_without_tool("job_properties")
     def test_import_export_failed_job(self):
-        history_name = "for_export_include_failed_job"
+        history_name = f"for_export_include_failed_job_{uuid4()}"
         history_id = self.dataset_populator.new_history(name=history_name)
         self.dataset_populator.run_tool_raw('job_properties', inputs={'failbool': True}, history_id=history_id)
         self.dataset_populator.wait_for_history(history_id, assert_ok=False)
@@ -317,7 +318,7 @@ class ImportExportTests(BaseHistories):
         self._check_imported_dataset(history_id=imported_history_id, hid=1, assert_ok=False, hda_checker=check_failed, job_checker=check_failed)
 
     def test_import_metadata_regeneration(self):
-        history_name = "for_import_metadata_regeneration"
+        history_name = f"for_import_metadata_regeneration_{uuid4()}"
         history_id = self.dataset_populator.new_history(name=history_name)
         self.dataset_populator.new_dataset(history_id, content=open(self.test_data_resolver.get_filename("1.bam"), 'rb'), file_type='bam', wait=True)
         imported_history_id = self._reimport_history(history_id, history_name)
@@ -337,13 +338,14 @@ class ImportExportTests(BaseHistories):
         self.dataset_populator.wait_for_history_jobs(imported_history_id, assert_ok=True)
         bai_metadata = import_bam_metadata["meta_files"][0]
         assert bai_metadata["file_type"] == "bam_index"
+        assert 'api/' in bai_metadata["download_url"], bai_metadata["download_url"]
         api_url = bai_metadata["download_url"].split("api/", 1)[1]
         bai_response = self._get(api_url)
         self._assert_status_code_is(bai_response, 200)
         assert len(bai_response.content) > 4
 
     def test_import_export_collection(self):
-        history_name = "for_export_with_collections"
+        history_name = f"for_export_with_collections_{uuid4()}"
         history_id = self.dataset_populator.new_history(name=history_name)
         self.dataset_collection_populator.create_list_in_history(history_id, contents=["Hello", "World"], direct_upload=True)
 
@@ -365,7 +367,7 @@ class ImportExportTests(BaseHistories):
         self._check_imported_collection(imported_history_id, hid=1, collection_type="list", elements_checker=check_elements)
 
     def test_import_export_nested_collection(self):
-        history_name = "for_export_with_nested_collections"
+        history_name = f"for_export_with_nested_collections_{uuid4()}"
         history_id = self.dataset_populator.new_history(name=history_name)
         self.dataset_collection_populator.create_list_of_pairs_in_history(history_id)
 

lib/tool_shed/webapp/fast_app.py

@@ -1,5 +1,5 @@
+from a2wsgi import WSGIMiddleware
 from fastapi import FastAPI
-from fastapi.middleware.wsgi import WSGIMiddleware
 
 from galaxy.webapps.base.api import (
     add_exception_handler,

packages/webapps/requirements.txt

@@ -1,3 +1,4 @@
+a2wsgi
 galaxy-app
 Cheetah3
 fastapi>=0.68.2,!=0.69.0,!=0.70.0,!=0.70.1

pyproject.toml

@@ -14,6 +14,7 @@ name = "galaxyproject"
 url = "https://wheels.galaxyproject.org/simple"
 
 [tool.poetry.dependencies]
+a2wsgi = "*"
 aiofiles = "*"
 Babel = "*"
 bdbag = "*"

test/unit/app/managers/test_DatasetManager.py

@@ -1,6 +1,7 @@
 """
 """
 import unittest
+from unittest import mock
 
 import sqlalchemy
 
@@ -208,9 +209,7 @@ def testable_url_for(*a, **k):
     return f'(fake url): {a}, {k}'
 
 
-DatasetSerializer.url_for = staticmethod(testable_url_for)
-
-
+@mock.patch('galaxy.managers.datasets.DatasetSerializer.url_for', testable_url_for)
 class DatasetSerializerTestCase(BaseTestCase):
 
     def set_up_managers(self):

test/unit/app/managers/test_HDAManager.py

@@ -1,4 +1,5 @@
 import unittest
+from unittest import mock
 
 import sqlalchemy
 
@@ -346,9 +347,7 @@ def testable_url_for(*a, **k):
     return f'(fake url): {a}, {k}'
 
 
-hdas.HDASerializer.url_for = staticmethod(testable_url_for)
-
-
+@mock.patch('galaxy.managers.hdas.HDASerializer.url_for', testable_url_for)
 class HDASerializerTestCase(HDATestCase):
 
     def set_up_managers(self):

test/unit/app/managers/test_HDCAManager.py

@@ -1,4 +1,5 @@
 import unittest
+from unittest import mock
 
 from galaxy.managers import (
     collections,
@@ -57,9 +58,7 @@ def testable_url_for(*a, **k):
     return f'(fake url): {a}, {k}'
 
 
-hdcas.HDCASerializer.url_for = staticmethod(testable_url_for)
-
-
+@mock.patch('galaxy.managers.hdcas.HDCASerializer.url_for', testable_url_for)
 class HDCASerializerTestCase(HDCATestCase):
 
     def set_up_managers(self):

test/unit/app/managers/test_HistoryManager.py

@@ -390,10 +390,8 @@ def testable_url_for(*a, **k):
     return f'(fake url): {a}, {k}'
 
 
-HistorySerializer.url_for = staticmethod(testable_url_for)
-hdas.HDASerializer.url_for = staticmethod(testable_url_for)
-
-
+@mock.patch('galaxy.managers.histories.HistorySerializer.url_for', testable_url_for)
+@mock.patch('galaxy.managers.hdas.HDASerializer.url_for', testable_url_for)
 class HistorySerializerTestCase(BaseTestCase):
 
     def set_up_managers(self):

test/unit/webapps/test_send_file.py

@@ -1,8 +1,8 @@
 import tempfile
 
 import pytest
+from a2wsgi import WSGIMiddleware
 from fastapi.applications import FastAPI
-from fastapi.middleware.wsgi import WSGIMiddleware
 from fastapi.testclient import TestClient
 
 from galaxy.util.bunch import Bunch