Merge branch 'release_23.1' into dev

This commit is contained in:
mvdbeek
2023-08-31 13:04:47 +02:00
25 changed files with 185 additions and 221 deletions
@@ -31,6 +31,8 @@ describe("PgeList.vue", () => {
offset: 0,
sort_by: "update_time",
sort_desc: true,
show_published: false,
show_shared: true,
};
const publishedGridApiParams = {
...personalGridApiParams,
+5 -1
View File
@@ -177,7 +177,11 @@ export default {
},
computed: {
dataProviderParameters() {
const extraParams = { search: this.effectiveFilter };
const extraParams = {
search: this.effectiveFilter,
show_published: false,
show_shared: true,
};
if (this.published) {
extraParams.show_published = true;
extraParams.show_shared = false;
+1 -1
View File
@@ -13546,7 +13546,7 @@ export interface operations {
header?: {
"run-as"?: string;
};
/** @description The ID of the History. */
/** @description History ID or any string. */
/** @description The ID of the item (`HDA`/`HDCA`) contained in the history. */
/**
* @description The type of the target history element.
@@ -261,7 +261,7 @@ window.bundleEntries.jqplot_box = function (options) {
chart : options.chart,
dataset_id : dataset.id,
dataset_groups : dataset_groups,
targets : options.targets,
target : options.target,
makeConfig : function( groups, plot_config ){
var boundary = getDomains( groups, 'x' );
$.extend( true, plot_config, {
+5
View File
@@ -91,6 +91,7 @@ from galaxy.objectstore import (
)
from galaxy.queue_worker import (
GalaxyQueueWorker,
reload_toolbox,
send_local_control_task,
)
from galaxy.quota import (
@@ -773,6 +774,10 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
# Start web stack message handling
self.application_stack.register_postfork_function(self.application_stack.start)
self.application_stack.register_postfork_function(self.queue_worker.bind_and_start)
# Reload toolbox to pick up changes to toolbox made after master was ready
self.application_stack.register_postfork_function(
lambda: reload_toolbox(self, save_integrated_tool_panel=False), post_fork_only=True
)
# Delay toolbox index until after startup
self.application_stack.register_postfork_function(
lambda: send_local_control_task(self, "rebuild_toolbox_search_index")
+10 -3
View File
@@ -30,6 +30,8 @@ from galaxy.model.base import transaction
from galaxy.util import DEFAULT_SOCKET_TIMEOUT
from . import IdentityProvider
log = logging.getLogger(__name__)
# key: a component name which PSA requests.
# value: is the name of a class associated with that key.
DEFAULTS = {"STRATEGY": "Strategy", "STORAGE": "Storage"}
@@ -170,9 +172,14 @@ class PSAAuthnz(IdentityProvider):
if not user_authnz_token or not user_authnz_token.extra_data:
return False
# refresh tokens if they reached their half lifetime
if int(user_authnz_token.extra_data["auth_time"]) + int(user_authnz_token.extra_data["expires"]) / 2 <= int(
time.time()
):
if "expires" in user_authnz_token.extra_data:
expires = user_authnz_token.extra_data["expires"]
elif "expires_in" in user_authnz_token.extra_data:
expires = user_authnz_token.extra_data["expires_in"]
else:
log.debug("No `expires` or `expires_in` key found in token extra data, cannot refresh")
return False
if int(user_authnz_token.extra_data["auth_time"]) + int(expires) / 2 <= int(time.time()):
on_the_fly_config(trans.sa_session)
if self.config["provider"] == "azure":
self.refresh_azure(user_authnz_token)
-11
View File
@@ -159,17 +159,6 @@ class ProvidesAppContext:
"""
return self.app.model.session
def expunge_all(self):
"""Expunge all the objects in Galaxy's SQLAlchemy sessions."""
app = self.app
context = app.model.context
context.expunge_all()
# This is a bit hacky, should refctor this. Maybe refactor to app -> expunge_all()
if hasattr(app, "install_model"):
install_model = app.install_model
if install_model != app.model:
install_model.context.expunge_all()
def get_toolbox(self):
"""Returns the application toolbox.
+14
View File
@@ -10746,6 +10746,20 @@ WorkflowInvocationStep.subworkflow_invocation_id = column_property(
# <user_obj>.preferences[pref_name] = pref_value
User.preferences = association_proxy("_preferences", "value", creator=UserPreference)
# Optimized version of getting the current Galaxy session.
# See https://github.com/sqlalchemy/sqlalchemy/discussions/7638 for approach
session_partition = select(
GalaxySession,
func.row_number().over(order_by=GalaxySession.update_time, partition_by=GalaxySession.user_id).label("index"),
).alias()
partitioned_session = aliased(GalaxySession, session_partition)
User.current_galaxy_session = relationship(
partitioned_session,
primaryjoin=and_(partitioned_session.user_id == User.id, session_partition.c.index < 2),
uselist=False,
viewonly=True,
)
@event.listens_for(HistoryDatasetCollectionAssociation, "init")
def receive_init(target, args, kwargs):
+1 -1
View File
@@ -238,7 +238,7 @@ def reload_tool_data_tables(app, **kwargs):
def rebuild_toolbox_search_index(app, **kwargs):
if app.is_webapp:
if app.is_webapp and app.database_heartbeat.is_config_watcher:
if app.toolbox_search.index_count < app.toolbox._reload_count:
app.reindex_tool_search()
else:
+4 -1
View File
@@ -53,7 +53,10 @@ class StepOrderIndexGetter(GetterDict):
if key == "workflow_step_id":
return self._obj.workflow_step.order_index
elif key == "dependent_workflow_step_id":
return self._obj.dependent_workflow_step.order_index
if self._obj.dependent_workflow_step_id:
return self._obj.dependent_workflow_step.order_index
else:
return default
return super().get(key, default)
+3
View File
@@ -923,6 +923,9 @@ def __parse_param_elem(param_elem, i=0):
else:
value = None
if value is None and attrib.get("location", None) is not None:
value = os.path.basename(attrib["location"])
children_elem = param_elem
if children_elem is not None:
# At this time, we can assume having children only
+36 -5
View File
@@ -6,6 +6,10 @@ import sqlite3
import tempfile
import zlib
from threading import Lock
from typing import (
Dict,
Optional,
)
from sqlitedict import SqliteDict
@@ -134,7 +138,7 @@ class ToolCache:
def __init__(self):
self._lock = Lock()
self._hash_by_tool_paths = {}
self._hash_by_tool_paths: Dict[str, ToolHash] = {}
self._tools_by_path = {}
self._tool_paths_by_id = {}
self._macro_paths_by_id = {}
@@ -195,9 +199,12 @@ class ToolCache:
try:
new_mtime = os.path.getmtime(config_filename)
tool_hash = self._hash_by_tool_paths.get(config_filename)
if tool_hash.modtime < new_mtime:
if md5_hash_file(config_filename) != tool_hash.hash:
if tool_hash and tool_hash.modtime_less_than(new_mtime):
if not tool_hash.hash_equals(md5_hash_file(config_filename)):
return True
else:
# No change of content, so not necessary to calculate the md5 checksum every time
tool_hash.modtime = new_mtime
tool = self._tools_by_path[config_filename]
for macro_path in tool._macro_paths:
new_mtime = os.path.getmtime(macro_path)
@@ -256,13 +263,37 @@ class ToolCache:
class ToolHash:
def __init__(self, path, modtime=None, lazy_hash=False):
def __init__(self, path: str, modtime: Optional[float] = None, lazy_hash: bool = False):
self.path = path
self.modtime = modtime or os.path.getmtime(path)
self._modtime = modtime or os.path.getmtime(path)
self._tool_hash = None
if not lazy_hash:
self.hash # noqa: B018
def modtime_less_than(self, other_modtime: float):
if self._modtime is None:
# For the purposes of the tool cache,
# if we haven't seen the modtime we consider it not equal
return True
return self._modtime < other_modtime
def hash_equals(self, other_hash: Optional[str]):
if self._tool_hash is None or other_hash is None:
# For the purposes of the tool cache,
# if we haven't seen the hash yet we consider it not equal
return False
return self.hash == other_hash
@property
def modtime(self) -> float:
if self._modtime is None:
self._modtime = os.path.getmtime(self.path)
return self._modtime
@modtime.setter
def modtime(self, new_value: float):
self._modtime = new_value
@property
def hash(self):
if self._tool_hash is None:
+16 -7
View File
@@ -184,16 +184,25 @@ def _process_raw_inputs(
param_value = raw_input_dict["value"]
param_extra = raw_input_dict["attributes"]
location = param_extra.get("location")
if param_value is None and location:
# If no value is given, we try to get the file name directly from the URL
param_value = os.path.basename(location)
if not value.type == "text":
param_value = _split_if_str(param_value)
if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter):
if not isinstance(param_value, list):
param_value = [param_value]
for v in param_value:
_add_uploaded_dataset(context.for_state(), v, param_extra, value, required_files)
if location and value.multiple:
# We get the input/s from the location which can be a list of urls separated by commas
locations = _split_if_str(location)
param_value = []
for location in locations:
v = os.path.basename(location)
param_value.append(v)
# param_extra should contain only the corresponding location
extra = dict(param_extra)
extra["location"] = location
_add_uploaded_dataset(context.for_state(), v, extra, value, required_files)
else:
if not isinstance(param_value, list):
param_value = [param_value]
for v in param_value:
_add_uploaded_dataset(context.for_state(), v, param_extra, value, required_files)
processed_value = param_value
elif isinstance(value, galaxy.tools.parameters.basic.DataCollectionToolParameter):
assert "collection" in param_extra
+7 -6
View File
@@ -43,8 +43,9 @@ class ApplicationStack:
return {}
@classmethod
def register_postfork_function(cls, f, *args, **kwargs):
f(*args, **kwargs)
def register_postfork_function(cls, f, *args, post_fork_only=False, **kwargs):
if not post_fork_only:
f(*args, **kwargs)
def __init__(self, app=None, config=None):
self.app = app
@@ -191,7 +192,7 @@ class GunicornApplicationStack(ApplicationStack):
late_postfork_thread: threading.Thread
@classmethod
def register_postfork_function(cls, f, *args, **kwargs):
def register_postfork_function(cls, f, *args, post_fork_only=False, **kwargs):
# do_post_fork determines if we need to run postfork functions
if cls.do_post_fork:
# if so, we call ApplicationStack.late_postfork once after forking ...
@@ -199,7 +200,7 @@ class GunicornApplicationStack(ApplicationStack):
os.register_at_fork(after_in_child=cls.late_postfork)
# ... and store everything we need to run in ApplicationStack.postfork_functions
cls.postfork_functions.append(lambda: f(*args, **kwargs))
else:
elif not post_fork_only:
f(*args, **kwargs)
@classmethod
@@ -300,8 +301,8 @@ def application_stack_log_formatter():
return logging.Formatter(fmt=application_stack_class().log_format)
def register_postfork_function(f, *args, **kwargs):
application_stack_class().register_postfork_function(f, *args, **kwargs)
def register_postfork_function(f, *args, post_fork_only=False, **kwargs):
application_stack_class().register_postfork_function(f, *args, post_fork_only=post_fork_only**kwargs)
def get_app_kwds(config_section, app_name=None):
+1 -2
View File
@@ -306,7 +306,6 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, context.ProvidesHistoryCo
self.user_manager = app[UserManager]
self.session_manager = app[GalaxySessionManager]
super().__init__(environ)
self.expunge_all()
config = self.app.config
self.debug = asbool(config.get("debug", False))
x_frame_options = getattr(config, "x_frame_options", None)
@@ -784,7 +783,7 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, context.ProvidesHistoryCo
history = None
set_permissions = False
try:
users_last_session = user.galaxy_sessions[0]
users_last_session = user.current_galaxy_session
except Exception:
users_last_session = None
if (
@@ -838,7 +838,7 @@ class FastAPIHistoryContents:
self,
response: Response,
trans: ProvidesHistoryContext = DependsOnTrans,
history_id: DecodedDatabaseIdField = HistoryIDPathParam,
history_id: str = Path(..., description="History ID or any string."),
id: DecodedDatabaseIdField = HistoryItemIDPathParam,
type: HistoryContentType = ContentTypePathParam,
serialization_params: SerializationParams = Depends(query_serialization_params),
-6
View File
@@ -1219,12 +1219,6 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf):
normalize_remote_user_email=conf.get("normalize_remote_user_email", False),
),
)
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get("use_recursive", True)):
from paste import recursive
app = wrap_if_allowed(app, stack, recursive.RecursiveMiddleware, args=(conf,))
# Error middleware
app = wrap_if_allowed(app, stack, ErrorMiddleware, args=(conf,))
@@ -77,7 +77,7 @@ class UserListGrid(grids.Grid):
class LastLoginColumn(grids.GridColumn):
def get_value(self, trans, grid, user):
if user.galaxy_sessions:
return self.format(user.galaxy_sessions[0].update_time)
return self.format(user.current_galaxy_session.update_time)
return "never"
def sort(self, trans, query, ascending, column_name=None):
@@ -253,13 +253,17 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
# datatype changing
datatype_options = [(ext_name, ext_id) for ext_id, ext_name in ldatatypes]
datatype_disable = len(datatype_options) == 0
datatype_input_default_value = None
current_datatype = trans.app.datatypes_registry.datatypes_by_extension.get(data.ext)
if current_datatype and current_datatype.is_datatype_change_allowed():
datatype_input_default_value = data.ext
datatype_inputs = [
{
"type": "select",
"name": "datatype",
"label": "New Type",
"options": datatype_options,
"value": [ext_id for ext_id, ext_name in ldatatypes if ext_id == data.ext],
"value": datatype_input_default_value,
"help": "This will change the datatype of the existing dataset but not modify its contents. Use this if Galaxy has incorrectly guessed the type of your dataset.",
}
]
-6
View File
@@ -86,12 +86,6 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf):
# wrapped around the application (it can interact poorly with
# other middleware):
app = wrap_if_allowed(app, stack, httpexceptions.make_middleware, name="paste.httpexceptions", args=(conf,))
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get("use_recursive", True)):
from paste import recursive
app = wrap_if_allowed(app, stack, recursive.RecursiveMiddleware, args=(conf,))
# Error middleware
app = wrap_if_allowed(app, stack, ErrorMiddleware, args=(conf,))
@@ -170,8 +170,9 @@ class Users(BaseUIController, ReportQueryBuilder):
.filter(galaxy.model.User.table.c.deleted == false())
.order_by(galaxy.model.User.table.c.email)
):
if user.galaxy_sessions:
last_galaxy_session = user.galaxy_sessions[0]
current_galaxy_session = user.current_galaxy_session
if current_galaxy_session:
last_galaxy_session = current_galaxy_session
if last_galaxy_session.update_time < cutoff_time:
users.append((user.email, last_galaxy_session.update_time.strftime("%Y-%m-%d")))
else:
+2 -2
View File
@@ -46,8 +46,8 @@ class UserGrid(grids.Grid):
class LastLoginColumn(grids.GridColumn):
def get_value(self, trans, grid, user):
if user.galaxy_sessions:
return self.format(user.galaxy_sessions[0].update_time)
if user.current_galaxy_session:
return self.format(user.current_galaxy_session.update_time)
return "never"
class StatusColumn(grids.GridColumn):
-5
View File
@@ -259,12 +259,7 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf):
normalize_remote_user_email=conf.get("normalize_remote_user_email", False),
),
)
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get("use_recursive", True)):
from paste import recursive
app = wrap_if_allowed(app, stack, recursive.RecursiveMiddleware, args=(conf,))
# Transaction logging (apache access.log style)
if asbool(conf.get("use_translogger", True)):
from paste.translogger import TransLogger
+67 -11
View File
@@ -31,16 +31,6 @@ cp '$isee_script' '$outfile' &&
<configfiles>
<configfile name="isee_script"><![CDATA[
## Import render function to template R Shiny app code
## -----------------------------------------------------------------------------
#import os
#import importlib.util
#set modpath = $os.path.join($__tool_directory__, "isee/render.py")
#set spec = $importlib.util.spec_from_file_location("render", $modpath)
#set render = $importlib.util.module_from_spec(spec)
#$spec.loader.exec_module(render)
## Stop warning messages being emitted from R while still allowing genuine job failure
## -----------------------------------------------------------------------------
devNull <- file("/dev/null", open = "wt")
@@ -54,8 +44,74 @@ library(HDF5Array)
sce_path <- 'sce'
sce <- loadHDF5SummarizedExperiment(sce_path)
sce <- registerAppOptions(sce, color.maxlevels=40)
$render.app()
categorical_color_fun <- function(n){
if (n <= 37) {
# Less than 37 colours, use something from colour brewer
# (joining a bunch of palettes, best colours up front)
multiset <- c(
RColorBrewer::brewer.pal(9, "Set1"),
RColorBrewer::brewer.pal(8, "Set2"),
RColorBrewer::brewer.pal(12, "Set3"),
RColorBrewer::brewer.pal(8, "Dark2"))
return(multiset[1:n])
}
else {
# More that 37, well at least it looks pretty
return(rainbow(n))
}
}
ecm <- ExperimentColorMap(
# The default is viridis::viridis
# https://cran.r-project.org/web/packages/viridis/vignettes/intro-to-viridis.html#the-color-scales
# Setting continous is entirely a matter of taste
# Some find magma easier to read than viridis
all_continuous = list(
assays = viridis::magma,
colData = viridis::magma,
rowData = viridis::magma
),
all_discrete = list(
colData = categorical_color_fun,
rowData = categorical_color_fun
)
)
# These options are all sce-contents agnostic.
initial_plots <- c(
# Show umap with clusters by default
ReducedDimensionPlot(
DataBoxOpen=TRUE,
ColorBy="Column data",
VisualBoxOpen=TRUE,
PanelWidth=6L),
# Show gene expression plot separated (and coloured) by cluster, by default.
FeatureAssayPlot(XAxis = "Column data",
DataBoxOpen=TRUE,
VisualBoxOpen=TRUE,
ColorBy="Column data",
PanelWidth=6L
),
# Gene list is better wide
RowDataTable(PanelWidth=12L),
# For cell level observations (QC.)
ColumnDataPlot(PanelWidth=6L,
DataBoxOpen=TRUE,
VisualBoxOpen=TRUE )
)
app <- iSEE(sce,
colormap=ecm,
initial=initial_plots)
shiny::runApp(app, host="0.0.0.0", port=8888, quiet=TRUE, launch.browser=FALSE)
-147
View File
@@ -1,147 +0,0 @@
"""Pre-configured iSEE options for selection by user.
Currently user configuration is NOT IMPLEMENTED - the code is here but option
is hidden from user in the tool form. Without ``custom`` selection, this simply
returns a DEFAULT iSEE configuration (defined at the bottom of this file).
These are preconfigured iSEE parameters that can be chosen by the user
as multiple choice (checkbox/select) input fields. Each parameter
(e.g. "initial") has a list of options whose indices should match the value of
the input field in the tool XML.
In all likelihood, the majority of options in
iSEE don't make sense without first being able to sniff the data and so are not
feasible to expose in the Galaxy tool form. Currently you will see that only
plot type and width can be exposed.
"""
def app():
"""Render R code to create iSEE app from user input."""
return DEFAULT
def render_plots(call, plots):
"""Render plot calls from user input."""
if not plots:
return call
plot_calls_list = [
get_render_func(plot)(
# user plot params as kwargs here
)
for plot in plots
]
plot_calls = ",\ninitial=c(\n" + ",\n".join(plot_calls_list) + ")"
return call + plot_calls
def get_render_func(plot):
"""Return the appropriate function to render plot."""
# This is probably broken and unused
return OPTIONS["plots"][plot["plot_types"]["plot_type"].value] # type: ignore[index]
def reduced_dimension_plot(pw="6L"):
"""Render a ReducedDimensionPlot object call."""
return f"""ReducedDimensionPlot(
PanelWidth={pw})"""
def feature_assay_plot(pw="6L"):
"""Render a FeatureAssayPlot object call."""
return f"""FeatureAssayPlot(
PanelWidth={pw})"""
def row_data_table(pw="12L"):
"""Render a RowDataTable object call."""
return f"RowDataTable(PanelWidth={pw})"
def column_data_plot(pw="6L"):
"""Render a ColumnDataPlot object call."""
return f"ColumnDataPlot(PanelWidth={pw})"
OPTIONS = {
"plots": {
"reduced_dimension_plot": reduced_dimension_plot,
"feature_assay_plot": feature_assay_plot,
"row_data_table": row_data_table,
"column_data_plot": column_data_plot,
},
"colormaps": {},
"extra": {},
}
DEFAULT = """
sce <- registerAppOptions(sce, color.maxlevels=40)
categorical_color_fun <- function(n){
if (n <= 37) {
# Less than 37 colours, use something from colour brewer
# (joining a bunch of palettes, best colours up front)
multiset <- c(
RColorBrewer::brewer.pal(9, "Set1"),
RColorBrewer::brewer.pal(8, "Set2"),
RColorBrewer::brewer.pal(12, "Set3"),
RColorBrewer::brewer.pal(8, "Dark2"))
return(multiset[1:n])
}
else {
# More that 37, well at least it looks pretty
return(rainbow(n))
}
}
ecm <- ExperimentColorMap(
# The default is viridis::viridis
# https://cran.r-project.org/web/packages/viridis/vignettes/intro-to-viridis.html#the-color-scales
# Setting continous is entirely a matter of taste
# Some find magma easier to read than viridis
all_continuous = list(
assays = viridis::magma,
colData = viridis::magma,
rowData = viridis::magma
),
all_discrete = list(
colData = categorical_color_fun,
rowData = categorical_color_fun
)
)
# These options are all sce-contents agnostic.
initial_plots <- c(
# Show umap with clusters by default
ReducedDimensionPlot(
DataBoxOpen=TRUE,
ColorBy="Column data",
VisualBoxOpen=TRUE,
PanelWidth=6L),
# Show gene expression plot separated (and coloured) by cluster, by default.
FeatureAssayPlot(XAxis = "Column data",
DataBoxOpen=TRUE,
VisualBoxOpen=TRUE,
ColorBy="Column data",
PanelWidth=6L
),
# Gene list is better wide
RowDataTable(PanelWidth=12L),
# For cell level observations (QC.)
ColumnDataPlot(PanelWidth=6L,
DataBoxOpen=TRUE,
VisualBoxOpen=TRUE )
)
app <- iSEE(sce,
colormap=ecm,
initial=initial_plots)
"""