Merge pull request #18279 from mvdbeek/server_external_hgweb

[23.1] More fixes for running the TS with external hgweb

lib/galaxy/web/framework/middleware/error.py

@@ -9,6 +9,7 @@ Error handler middleware
 When an exception is thrown from the wrapper application, this logs
 the exception and displays an error page.
 """
+import logging
 import sys
 import traceback
 from io import StringIO
@@ -25,6 +26,8 @@ from paste.exceptions import (
     reporter,
 )
 
+log = logging.getLogger(__name__)
+
 __all__ = ("ErrorMiddleware", "handle_exception")
 
 
@@ -170,10 +173,11 @@ class ErrorMiddleware:
                 for expect in environ.get("paste.expected_exceptions", []):
                     if isinstance(exc_info[1], expect):
                         raise
+                log.exception("Uncaught Exception")
                 start_response("500 Internal Server Error", [("content-type", "text/html")], exc_info)
                 # @@: it would be nice to deal with bad content types here
                 response = self.exception_handler(exc_info, environ)
-                return [response]
+                return [response.encode(errors="ignore")]
             finally:
                 # clean up locals...
                 exc_info = None
@@ -184,7 +188,7 @@ class ErrorMiddleware:
             return app_iter
         return CatchingIter(app_iter, environ, sr_checker, self)
 
-    def exception_handler(self, exc_info, environ):
+    def exception_handler(self, exc_info, environ) -> str:
         simple_html_error = False
         if self.xmlhttp_key:
             get_vars = wsgilib.parse_querystring(environ)
@@ -193,7 +197,6 @@ class ErrorMiddleware:
         return handle_exception(
             exc_info,
             environ["wsgi.errors"],
-            html=True,
             debug_mode=self.debug_mode,
             error_email=self.error_email,
             error_log=self.error_log,
@@ -344,7 +347,6 @@ class Supplement:
 def handle_exception(
     exc_info,
     error_stream,
-    html=True,
     debug_mode=False,
     error_email=None,
     error_log=None,
@@ -358,7 +360,7 @@ def handle_exception(
     error_message=None,
     simple_html_error=False,
     environ=None,
-):
+) -> str:
     """
     For exception handling outside of a web context
 
@@ -389,54 +391,51 @@ def handle_exception(
             smtp_use_tls=smtp_use_tls,
             subject_prefix=error_subject_prefix,
         )
-        rep_err = send_report(rep, exc_data, html=html)
+        rep_err = send_report(rep, exc_data, html=True)
         if rep_err:
             extra_data += rep_err
         else:
             reported = True
     if error_log:
         rep = reporter.LogReporter(filename=error_log)
-        rep_err = send_report(rep, exc_data, html=html)
+        rep_err = send_report(rep, exc_data, html=True)
         if rep_err:
             extra_data += rep_err
         else:
             reported = True
     if show_exceptions_in_wsgi_errors:
         rep = reporter.FileReporter(file=error_stream)
-        rep_err = send_report(rep, exc_data, html=html)
+        rep_err = send_report(rep, exc_data, html=True)
         if rep_err:
             extra_data += rep_err
         else:
             reported = True
     else:
         error_stream.write(f"Error - {exc_data.exception_type}: {exc_data.exception_value}\n")
-    if html:
-        if debug_mode and simple_html_error:
-            return_error = formatter.format_html(
-                exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False
-            )
-            reported = True
-        elif debug_mode and not simple_html_error:
-            error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False)
-            head_html = formatter.error_css + formatter.hide_display_js
-            return_error = error_template(head_html, error_html, extra_data)
-            extra_data = ""
-            reported = True
-        else:
-            msg = (
-                error_message
-                or """
-            An error occurred.
-            """
-            )
-            extra = "<p><b>The error has been logged to our team.</b>"
-            if "sentry_event_id" in environ:
-                extra += " If you want to contact us about this error, please reference the following<br><br>"
-                extra += f"<b><large>GURU MEDITATION: #{environ['sentry_event_id']}</large></b>"
-            extra += "</p>"
-            return_error = error_template("", msg, extra)
+    if debug_mode and simple_html_error:
+        return_error = formatter.format_html(
+            exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False
+        )
+        reported = True
+    elif debug_mode and not simple_html_error:
+        error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False)
+        head_html = formatter.error_css + formatter.hide_display_js
+        return_error = error_template(head_html, error_html, extra_data)
+        extra_data = ""
+        reported = True
     else:
-        return_error = None
+        msg = (
+            error_message
+            or """
+        An error occurred.
+        """
+        )
+        extra = "<p><b>The error has been logged to our team.</b>"
+        if "sentry_event_id" in environ:
+            extra += " If you want to contact us about this error, please reference the following<br><br>"
+            extra += f"<b><large>GURU MEDITATION: #{environ['sentry_event_id']}</large></b>"
+        extra += "</p>"
+        return_error = error_template("", msg, extra)
     if not reported and error_stream:
         err_report = formatter.format_text(exc_data, show_hidden_frames=True)
         err_report += f"\n{'-' * 60}\n"

lib/tool_shed/util/hg_util.py

@@ -71,14 +71,13 @@ def get_hgrc_path(repo_path):
     return os.path.join(repo_path, ".hg", "hgrc")
 
 
-def create_hgrc_file(app, repository):
+def create_hgrc_file(app, repository, repo_path):
     # Since we support both http and https, we set `push_ssl` to False to
     # override the default (which is True) in the Mercurial API.
     # The hg purge extension purges all files and directories not being tracked
     # by Mercurial in the current repository. It will remove unknown files and
     # empty directories. This is not currently used because it is not supported
     # in the Mercurial API.
-    repo_path = repository.repo_path(app)
     hgrc_path = get_hgrc_path(repo_path)
     with open(hgrc_path, "w") as fp:
         fp.write("[paths]\n")

lib/tool_shed/util/hgweb_config.py

@@ -5,8 +5,6 @@ import shutil
 import threading
 from datetime import date
 
-from galaxy.util import unicodify
-
 log = logging.getLogger(__name__)
 
 new_hgweb_config_template = """
@@ -20,6 +18,7 @@ class HgWebConfigManager:
         self.hgweb_config_dir = None
         self.in_memory_config = None
         self.lock = threading.Lock()
+        self.hgweb_repo_prefix = None
 
     def add_entry(self, lhs, rhs):
         """Add an entry in the hgweb.config file for a new repository."""
@@ -35,8 +34,8 @@ class HgWebConfigManager:
             self.in_memory_config.set("paths", lhs, rhs)
             # Persist our in-memory configuration.
             self.write_config()
-        except Exception as e:
-            log.debug("Exception in HgWebConfigManager.add_entry(): %s", unicodify(e))
+        except Exception:
+            log.exception("Exception in HgWebConfigManager.add_entry()")
         finally:
             self.lock.release()
 
@@ -51,8 +50,8 @@ class HgWebConfigManager:
             self.in_memory_config.set("paths", new_lhs, new_rhs)
             # Persist our in-memory configuration.
             self.write_config()
-        except Exception as e:
-            log.debug("Exception in HgWebConfigManager.change_entry(): %s", unicodify(e))
+        except Exception:
+            log.exception("Exception in HgWebConfigManager.change_entry()")
         finally:
             self.lock.release()
 

lib/tool_shed/util/repository_util.py

@@ -2,6 +2,7 @@ import configparser
 import logging
 import os
 import re
+import tempfile
 from typing import (
     Optional,
     Tuple,
@@ -154,23 +155,18 @@ def create_repo_info_dict(
 def create_repository_admin_role(app: "ToolShedApp", repository: "Repository"):
     """
     Create a new role with name-spaced name based on the repository name and its owner's public user
-    name.  This will ensure that the tole name is unique.
+    name.  This will ensure that the role name is unique.
     """
     sa_session = app.model.session
     name = get_repository_admin_role_name(str(repository.name), str(repository.user.username))
     description = "A user or group member with this role can administer this repository."
     role = app.model.Role(name=name, description=description, type=app.model.Role.types.SYSTEM)
     sa_session.add(role)
-    session = sa_session()
-    with transaction(session):
-        session.commit()
     # Associate the role with the repository owner.
     app.model.UserRoleAssociation(repository.user, role)
     # Associate the role with the repository.
     rra = app.model.RepositoryRoleAssociation(repository, role)
     sa_session.add(rra)
-    with transaction(session):
-        session.commit()
     return role
 
 
@@ -180,7 +176,7 @@ def create_repository(
     type: str,
     description,
     long_description,
-    user_id,
+    user,
     category_ids=None,
     remote_repository_url=None,
     homepage_url=None,
@@ -196,43 +192,40 @@ def create_repository(
         homepage_url=homepage_url,
         description=description,
         long_description=long_description,
-        user_id=user_id,
+        user=user,
     )
-    # Flush to get the id.
     sa_session.add(repository)
-    session = sa_session()
-    with transaction(session):
-        session.commit()
-    # Create an admin role for the repository.
-    create_repository_admin_role(app, repository)
-    # Determine the repository's repo_path on disk.
-    dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id))
-    # Create directory if it does not exist.
-    if not os.path.exists(dir):
-        os.makedirs(dir)
-    # Define repo name inside hashed directory.
-    repository_path = os.path.join(dir, "repo_%d" % repository.id)
-    # Create local repository directory.
-    if not os.path.exists(repository_path):
-        os.makedirs(repository_path)
-    # Create the local repository.
-    init_repository(repo_path=repository_path)
-    # Add an entry in the hgweb.config file for the local repository.
-    lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"
-    app.hgweb_config_manager.add_entry(lhs, repository_path)
-    # Create a .hg/hgrc file for the local repository.
-    create_hgrc_file(app, repository)
-    flush_needed = False
     if category_ids:
         # Create category associations
         for category_id in category_ids:
             category = sa_session.query(app.model.Category).get(app.security.decode_id(category_id))
             rca = app.model.RepositoryCategoryAssociation(repository, category)
             sa_session.add(rca)
-            flush_needed = True
-    if flush_needed:
-        with transaction(session):
-            session.commit()
+    # Create an admin role for the repository.
+    create_repository_admin_role(app, repository)
+    # Create a temporary repo_path on disk.
+    repository_path = tempfile.mkdtemp(
+        dir=app.config.file_path,
+        prefix=f"{repository.user.username}-{repository.name}",
+    )
+    # Create the local repository.
+    init_repository(repo_path=repository_path)
+    # Create a .hg/hgrc file for the local repository.
+    create_hgrc_file(app, repository, repo_path=repository_path)
+    # Add an entry in the hgweb.config file for the local repository.
+    lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"
+    # Flush to get the id.
+    session = sa_session()
+    with transaction(session):
+        session.commit()
+    dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id))
+    # Define repo name inside hashed directory.
+    final_repository_path = os.path.join(dir, "repo_%d" % repository.id)
+    # Create final repository directory.
+    if not os.path.exists(final_repository_path):
+        os.makedirs(final_repository_path)
+    os.rename(repository_path, final_repository_path)
+    app.hgweb_config_manager.add_entry(lhs, final_repository_path)
     # Update the repository registry.
     app.repository_registry.add_entry(repository)
     message = f"Repository <b>{escape(str(repository.name))}</b> has been created."
@@ -505,8 +498,8 @@ def update_repository(trans: "ProvidesUserContext", id: str, **kwds) -> Tuple[Op
 
         repo_dir = repository.repo_path(app)
         # Change the entry in the hgweb.config file for the repository.
-        old_lhs = f"repos/{repository.user.username}/{repository.name}"
-        new_lhs = f"repos/{repository.user.username}/{kwds['name']}"
+        old_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"
+        new_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{kwds['name']}"
         trans.app.hgweb_config_manager.change_entry(old_lhs, new_lhs, repo_dir)
 
         # Change the entry in the repository's hgrc file.

lib/tool_shed/util/shed_index.py

@@ -33,7 +33,7 @@ def _get_or_create_index(whoosh_index_dir):
     return get_or_create_index(whoosh_index_dir, repo_schema), get_or_create_index(tool_index_dir, tool_schema)
 
 
-def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs):
+def build_index(whoosh_index_dir, file_path, hgweb_config_dir, hgweb_repo_prefix, dburi, **kwargs):
     """
     Build two search indexes simultaneously
     One is for repositories and the other for tools.
@@ -51,7 +51,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs):
 
     execution_timer = ExecutionTimer()
     with repo_index.searcher() as searcher:
-        for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):
+        for repo in get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs):
             tools_list = repo.pop("tools_list")
             repo_id = repo["id"]
             indexed_document = searcher.document(id=repo_id)
@@ -85,7 +85,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs):
     return repos_indexed, tools_indexed
 
 
-def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):
+def get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs):
     """
     Load repos from DB and included tools from .xml configs.
     """
@@ -126,7 +126,7 @@ def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):
 
         # Load all changesets of the repo for lineage.
         repo_path = os.path.join(
-            hgweb_config_dir, hgwcm.get_entry(os.path.join("repos", repo.user.username, repo.name))
+            hgweb_config_dir, hgwcm.get_entry(os.path.join(hgweb_repo_prefix, repo.user.username, repo.name))
         )
         hg_repo = hg.repository(ui.ui(), repo_path.encode("utf-8"))
         lineage = []

lib/tool_shed/webapp/api/repositories.py

@@ -986,7 +986,7 @@ class RepositoriesController(BaseAPIController):
             type=repo_type,
             description=synopsis,
             long_description=description,
-            user_id=trans.user.id,
+            user=trans.user,
             category_ids=category_ids,
             remote_repository_url=remote_repository_url,
             homepage_url=homepage_url,

lib/tool_shed/webapp/api/tools.py

@@ -35,6 +35,7 @@ class ToolsController(BaseAPIController):
             trans.app.config.whoosh_index_dir,
             trans.app.config.file_path,
             trans.app.config.hgweb_config_dir,
+            trans.app.config.hgweb_repo_prefix,
             trans.app.config.database_connection,
         )
         return {

lib/tool_shed/webapp/app.py

@@ -101,6 +101,7 @@ class UniverseApplication(ToolShedApp, SentryClientMixin, HaltableContainer):
         # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located.
         self.hgweb_config_manager = hgweb_config_manager
         self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
+        self.hgweb_config_manager.hgweb_repo_prefix = self.config.hgweb_repo_prefix
         # Initialize the repository registry.
         self.repository_registry = tool_shed.repository_registry.Registry(self)
         # Configure Sentry client if configured

lib/tool_shed/webapp/controllers/repository.py

@@ -759,7 +759,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
                     repository_type,
                     description,
                     long_description,
-                    user_id=trans.user.id,
+                    user=trans.user,
                     category_ids=category_ids,
                     remote_repository_url=remote_repository_url,
                     homepage_url=homepage_url,

lib/tool_shed/webapp/model/__init__.py

@@ -143,6 +143,11 @@ class User(Base, Dictifiable):
         self.purged = False
         self.new_repo_alert = False
 
+    @property
+    def current_galaxy_session(self):
+        if self.galaxy_sessions:
+            return self.galaxy_sessions[0]
+
     def all_roles(self):
         roles = [ura.role for ura in self.roles]
         for group in [uga.group for uga in self.groups]:
@@ -417,12 +422,13 @@ class Repository(Base, Dictifiable):
     ]
     file_states = Bunch(NORMAL="n", NEEDS_MERGING="m", MARKED_FOR_REMOVAL="r", MARKED_FOR_ADDITION="a", NOT_TRACKED="?")
 
-    def __init__(self, private=False, times_downloaded=0, deprecated=False, **kwd):
+    def __init__(self, private=False, times_downloaded=0, deprecated=False, user=None, **kwd):
         super().__init__(**kwd)
         self.private = private
         self.times_downloaded = times_downloaded
         self.deprecated = deprecated
         self.name = self.name or "Unnamed repository"
+        self.user = user
 
     @property
     def hg_repo(self):
@@ -508,7 +514,9 @@ class Repository(Base, Dictifiable):
 
     def repo_path(self, app=None):
         # Keep app argument for compatibility with tool_shed_install Repository model
-        return hgweb_config_manager.get_entry(os.path.join("repos", self.user.username, self.name))
+        return hgweb_config_manager.get_entry(
+            os.path.join(hgweb_config_manager.hgweb_repo_prefix, self.user.username, self.name)
+        )
 
     def revision(self):
         repo = self.hg_repo

scripts/tool_shed/build_ts_whoosh_index.py

@@ -40,6 +40,7 @@ def parse_arguments():
     config = ts_config.ToolShedAppConfiguration(**app_properties)
     args.dburi = config.database_connection
     args.hgweb_config_dir = config.hgweb_config_dir
+    args.hgweb_repo_prefix = config.hgweb_repo_prefix
     args.whoosh_index_dir = config.whoosh_index_dir
     args.file_path = config.file_path
     if args.debug:

test/unit/shed_unit/test_shed_index.py

@@ -36,10 +36,11 @@ def community_file_dir():
 
 @pytest.fixture()
 def community_file_structure(community_file_dir):
-    community = namedtuple("community", "file_path hgweb_config_dir dburi")
+    community = namedtuple("community", "file_path hgweb_config_dir hgweb_repo_prefix dburi")
     return community(
         file_path=os.path.join(community_file_dir, "database", "community_files"),
         hgweb_config_dir=community_file_dir,
+        hgweb_repo_prefix="repos/",
         dburi="sqlite:///%s" % os.path.join(community_file_dir, "database", "community.sqlite"),
     )
 
@@ -49,6 +50,7 @@ def test_build_index(whoosh_index_dir, community_file_structure):
         whoosh_index_dir,
         community_file_structure.file_path,
         community_file_structure.hgweb_config_dir,
+        community_file_structure.hgweb_repo_prefix,
         community_file_structure.dburi,
     )
     assert repos_indexed == 1
@@ -59,6 +61,7 @@ def test_build_index(whoosh_index_dir, community_file_structure):
         whoosh_index_dir,
         community_file_structure.file_path,
         community_file_structure.hgweb_config_dir,
+        community_file_structure.hgweb_repo_prefix,
         community_file_structure.dburi,
     )
     assert repos_indexed == 0
@@ -74,6 +77,7 @@ def test_build_index(whoosh_index_dir, community_file_structure):
         whoosh_index_dir,
         community_file_structure.file_path,
         community_file_structure.hgweb_config_dir,
+        community_file_structure.hgweb_repo_prefix,
         community_file_structure.dburi,
     )
     assert repos_indexed == 1