Move short_term_storage related classes to its own path

The ShortTermStorageManager and related classes should live in their own package eventually and not within galaxy.web nor galaxy.managers as the short term storage microservices it's fairly independent and has way less dependencies than those other packages.

lib/galaxy/app.py

@@ -52,12 +52,6 @@ from galaxy.managers.library_datasets import LibraryDatasetsManager
 from galaxy.managers.notification import NotificationManager
 from galaxy.managers.roles import RoleManager
 from galaxy.managers.session import GalaxySessionManager
-from galaxy.managers.short_term_storage import (
-    ShortTermStorageAllocator,
-    ShortTermStorageConfiguration,
-    ShortTermStorageManager,
-    ShortTermStorageMonitor,
-)
 from galaxy.managers.tasks import (
     AsyncTasksManager,
     CeleryAsyncTasksManager,
@@ -113,6 +107,12 @@ from galaxy.security.vault import (
     Vault,
     VaultFactory,
 )
+from galaxy.short_term_storage import (
+    ShortTermStorageAllocator,
+    ShortTermStorageConfiguration,
+    ShortTermStorageManager,
+    ShortTermStorageMonitor,
+)
 from galaxy.tool_shed.cache import ToolShedRepositoryCache
 from galaxy.tool_shed.galaxy_install.client import InstallationTarget
 from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager

lib/galaxy/app_unittest_utils/galaxy_mock.py

@@ -26,12 +26,6 @@ from galaxy.managers.dbkeys import GenomeBuilds
 from galaxy.managers.hdas import HDAManager
 from galaxy.managers.histories import HistoryManager
 from galaxy.managers.jobs import JobSearch
-from galaxy.managers.short_term_storage import (
-    ShortTermStorageAllocator,
-    ShortTermStorageConfiguration,
-    ShortTermStorageManager,
-    ShortTermStorageMonitor,
-)
 from galaxy.managers.users import UserManager
 from galaxy.managers.workflows import WorkflowsManager
 from galaxy.model import tags
@@ -47,6 +41,12 @@ from galaxy.model.unittest_utils import (
     GalaxyDataTestConfig,
 )
 from galaxy.security import idencoding
+from galaxy.short_term_storage import (
+    ShortTermStorageAllocator,
+    ShortTermStorageConfiguration,
+    ShortTermStorageManager,
+    ShortTermStorageMonitor,
+)
 from galaxy.structured_app import (
     BasicSharedApp,
     MinimalManagerApp,

lib/galaxy/celery/tasks.py

@@ -32,7 +32,6 @@ from galaxy.managers.lddas import LDDAManager
 from galaxy.managers.markdown_util import generate_branded_pdf
 from galaxy.managers.model_stores import ModelStoreManager
 from galaxy.managers.notification import NotificationManager
-from galaxy.managers.short_term_storage import ShortTermStorageMonitor
 from galaxy.managers.tool_data import ToolDataImportManager
 from galaxy.metadata.set_metadata import set_metadata_portable
 from galaxy.model import (
@@ -59,6 +58,7 @@ from galaxy.schema.tasks import (
     WriteHistoryTo,
     WriteInvocationTo,
 )
+from galaxy.short_term_storage import ShortTermStorageMonitor
 from galaxy.structured_app import MinimalManagerApp
 from galaxy.tools import create_tool_from_representation
 from galaxy.tools.data_fetch import do_fetch

lib/galaxy/managers/collections.py

@@ -34,10 +34,6 @@ from galaxy.managers.hdas import (
 from galaxy.managers.hdcas import write_dataset_collection
 from galaxy.managers.histories import HistoryManager
 from galaxy.managers.lddas import LDDAManager
-from galaxy.managers.short_term_storage import (
-    ShortTermStorageMonitor,
-    storage_context,
-)
 from galaxy.model.base import transaction
 from galaxy.model.dataset_collections import builder
 from galaxy.model.dataset_collections.matching import MatchingCollections
@@ -47,6 +43,10 @@ from galaxy.model.mapping import GalaxyModelMapping
 from galaxy.schema.schema import DatasetCollectionInstanceType
 from galaxy.schema.tasks import PrepareDatasetCollectionDownload
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.short_term_storage import (
+    ShortTermStorageMonitor,
+    storage_context,
+)
 from galaxy.util import validation
 
 log = logging.getLogger(__name__)

lib/galaxy/managers/markdown_util.py

@@ -46,14 +46,14 @@ from galaxy.managers.jobs import (
     summarize_job_parameters,
 )
 from galaxy.managers.licenses import LicensesManager
-from galaxy.managers.short_term_storage import (
-    ShortTermStorageMonitor,
-    storage_context,
-)
 from galaxy.model.item_attrs import get_item_annotation_str
 from galaxy.model.orm.now import now
 from galaxy.schema import PdfDocumentType
 from galaxy.schema.tasks import GeneratePdfDownload
+from galaxy.short_term_storage import (
+    ShortTermStorageMonitor,
+    storage_context,
+)
 from galaxy.util.markdown import literal_via_fence
 from galaxy.util.resources import resource_string
 from galaxy.util.sanitize_html import sanitize_html

lib/galaxy/managers/model_stores.py

@@ -9,10 +9,6 @@ from galaxy.jobs.manager import JobManager
 from galaxy.managers.context import ProvidesUserContext
 from galaxy.managers.export_tracker import StoreExportTracker
 from galaxy.managers.histories import HistoryManager
-from galaxy.managers.short_term_storage import (
-    ShortTermStorageMonitor,
-    storage_context,
-)
 from galaxy.managers.users import UserManager
 from galaxy.model.base import transaction
 from galaxy.model.scoped_session import galaxy_scoped_session
@@ -43,6 +39,10 @@ from galaxy.schema.tasks import (
     WriteHistoryTo,
     WriteInvocationTo,
 )
+from galaxy.short_term_storage import (
+    ShortTermStorageMonitor,
+    storage_context,
+)
 from galaxy.structured_app import MinimalManagerApp
 from galaxy.version import VERSION
 

lib/galaxy/short_term_storage/__init__.py

@@ -1,3 +1,12 @@
+""" Short Term Storage service for Galaxy.
+
+This service is used to store files for a short period of time (e.g. a few minutes, hours or days)
+and then serve them to the client. This is useful for large files that take a long time to
+generate and are not worth storing permanently.
+"""
+
+# TODO: Move it to its own package https://github.com/galaxyproject/galaxy/pull/17058#discussion_r1400909897
+
 import abc
 import contextlib
 import json

lib/galaxy/webapps/galaxy/api/short_term_storage.py

@@ -3,7 +3,7 @@ API operations around galaxy.managers.short_term_storage infrastructure.
 """
 from uuid import UUID
 
-from galaxy.managers.short_term_storage import (
+from galaxy.short_term_storage import (
     ShortTermStorageMonitor,
     ShortTermStorageServeCancelledInformation,
     ShortTermStorageServeCompletedInformation,

lib/galaxy/webapps/galaxy/services/base.py

@@ -23,10 +23,6 @@ from galaxy.managers.base import (
 )
 from galaxy.managers.context import ProvidesUserContext
 from galaxy.managers.model_stores import create_objects_from_store
-from galaxy.managers.short_term_storage import (
-    ShortTermStorageAllocator,
-    ShortTermStorageTarget,
-)
 from galaxy.model import User
 from galaxy.model.store import (
     get_export_store_factory,
@@ -35,6 +31,10 @@ from galaxy.model.store import (
 from galaxy.schema.fields import EncodedDatabaseIdField
 from galaxy.schema.schema import AsyncTaskResultSummary
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.short_term_storage import (
+    ShortTermStorageAllocator,
+    ShortTermStorageTarget,
+)
 from galaxy.util import ready_name_for_url
 
 

lib/galaxy/webapps/galaxy/services/histories.py

@@ -41,7 +41,6 @@ from galaxy.managers.histories import (
     HistorySerializer,
 )
 from galaxy.managers.notification import NotificationManager
-from galaxy.managers.short_term_storage import ShortTermStorageAllocator
 from galaxy.managers.users import UserManager
 from galaxy.model import HistoryDatasetAssociation
 from galaxy.model.base import transaction
@@ -77,6 +76,7 @@ from galaxy.schema.tasks import (
 )
 from galaxy.schema.types import LatestLiteral
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.short_term_storage import ShortTermStorageAllocator
 from galaxy.util import restore_text
 from galaxy.webapps.galaxy.services.base import (
     async_task_summary,

lib/galaxy/webapps/galaxy/services/history_contents.py

@@ -57,7 +57,6 @@ from galaxy.managers.jobs import (
     summarize_jobs_to_dict,
 )
 from galaxy.managers.library_datasets import LibraryDatasetsManager
-from galaxy.managers.short_term_storage import ShortTermStorageAllocator
 from galaxy.model import (
     History,
     HistoryDatasetAssociation,
@@ -118,6 +117,7 @@ from galaxy.schema.tasks import (
     WriteHistoryContentTo,
 )
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.short_term_storage import ShortTermStorageAllocator
 from galaxy.util.zipstream import ZipstreamWrapper
 from galaxy.webapps.galaxy.services.base import (
     async_task_summary,

lib/galaxy/webapps/galaxy/services/invocations.py

@@ -23,7 +23,6 @@ from galaxy.exceptions import (
     ObjectNotFound,
 )
 from galaxy.managers.histories import HistoryManager
-from galaxy.managers.short_term_storage import ShortTermStorageAllocator
 from galaxy.managers.workflows import WorkflowsManager
 from galaxy.model import WorkflowInvocation
 from galaxy.model.store import (
@@ -45,6 +44,7 @@ from galaxy.schema.tasks import (
     WriteInvocationTo,
 )
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.short_term_storage import ShortTermStorageAllocator
 from galaxy.webapps.galaxy.services.base import (
     async_task_summary,
     ensure_celery_tasks_enabled,

lib/galaxy/webapps/galaxy/services/pages.py

@@ -13,7 +13,6 @@ from galaxy.managers.pages import (
     PageManager,
     PageSerializer,
 )
-from galaxy.managers.short_term_storage import ShortTermStorageAllocator
 from galaxy.model.base import transaction
 from galaxy.schema import PdfDocumentType
 from galaxy.schema.fields import DecodedDatabaseIdField
@@ -28,6 +27,7 @@ from galaxy.schema.schema import (
 )
 from galaxy.schema.tasks import GeneratePdfDownload
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.short_term_storage import ShortTermStorageAllocator
 from galaxy.webapps.galaxy.services.base import (
     async_task_summary,
     ensure_celery_tasks_enabled,

test/integration/test_celery_tasks.py

@@ -8,11 +8,11 @@ from galaxy.celery.tasks import (
     prepare_pdf_download,
     purge_hda,
 )
-from galaxy.managers.short_term_storage import ShortTermStorageAllocator
 from galaxy.model import HistoryDatasetAssociation
 from galaxy.schema import PdfDocumentType
 from galaxy.schema.schema import CreatePagePayload
 from galaxy.schema.tasks import GeneratePdfDownload
+from galaxy.short_term_storage import ShortTermStorageAllocator
 from galaxy_test.base.populators import (
     DatasetPopulator,
     wait_on,

test/unit/web_framework/test_short_term_storage.py

@@ -7,7 +7,7 @@ from galaxy.exceptions import (
     MessageException,
     ObjectNotFound,
 )
-from galaxy.managers.short_term_storage import (
+from galaxy.short_term_storage import (
     ShortTermStorageConfiguration,
     ShortTermStorageManager,
     ShortTermStorageServeCancelledInformation,

test/unit/webapps/test_service_base.py

@@ -2,8 +2,8 @@ from typing import Tuple
 
 import pytest
 
-from galaxy.managers.short_term_storage import ShortTermStorageAllocator
 from galaxy.schema.schema import ModelStoreFormat
+from galaxy.short_term_storage import ShortTermStorageAllocator
 from galaxy.webapps.galaxy.services.base import model_store_storage_target