opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-19 10:51:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release_21.05' into dev

Browse Source
This commit is contained in:
mvdbeek
2021-08-09 18:45:05 +02:00
parent 02e570d722 b38b41a1e4
commit ecc54ba00c
7 changed files with 179 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/integration_selenium.yaml
+1 -6
View File
@@ -5,8 +5,6 @@ concurrency:
cancel-in-progress: true
env:
GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
GALAXY_TEST_SELENIUM_REMOTE: '1'
GALAXY_TEST_SELENIUM_REMOTE_PORT: "4444"
GALAXY_SKIP_CLIENT_BUILD: '0'
GALAXY_TEST_SELENIUM_RETRIES: 1
YARN_INSTALL_OPTS: --frozen-lockfile
@@ -26,10 +24,6 @@ jobs:
POSTGRES_DB: postgres
ports:
- 5432:5432
selenium:
image: selenium/standalone-chrome:3.141.59
ports:
- 4444:4444
steps:
- name: Prune unused docker image, volumes and containers
run: docker system prune -a -f
@@ -48,6 +42,7 @@ jobs:
- uses: mvdbeek/gha-yarn-cache@master
with:
yarn-lock-file: 'galaxy root/client/yarn.lock'
- uses: nanasess/setup-chromedriver@master
- name: Run tests
run: './run_tests.sh -integration test/integration_selenium'
working-directory: 'galaxy root'
client/src/components/User/UserPreferencesModel.js
+7 -6
View File
@@ -1,15 +1,16 @@
import { getGalaxyInstance } from "app";
import _l from "utils/localization";
export const getUserPreferencesModel = () => {
export const getUserPreferencesModel = (user_id) => {
const Galaxy = getGalaxyInstance();
const config = Galaxy.config;
user_id = user_id || Galaxy.user.id;
return {
information: {
title: _l("Manage Information"),
id: "edit-preferences-information",
description: "Edit your email, addresses and custom parameters or change your public name.",
url: `api/users/${Galaxy.user.id}/information/inputs`,
url: `api/users/${user_id}/information/inputs`,
icon: "fa-user",
redirect: "user",
shouldRender: !config.use_remote_user && config.enable_account_interface,
@@ -19,7 +20,7 @@ export const getUserPreferencesModel = () => {
id: "edit-preferences-password",
description: _l("Allows you to change your login credentials."),
icon: "fa-unlock-alt",
url: `api/users/${Galaxy.user.id}/password/inputs`,
url: `api/users/${user_id}/password/inputs`,
submit_title: "Save Password",
redirect: "user",
shouldRender: !config.use_remote_user && config.enable_account_interface,
@@ -38,7 +39,7 @@ export const getUserPreferencesModel = () => {
id: "edit-preferences-permissions",
description:
"Grant others default access to newly created histories. Changes made here will only affect histories created after these settings have been stored.",
url: `api/users/${Galaxy.user.id}/permissions/inputs`,
url: `api/users/${user_id}/permissions/inputs`,
icon: "fa-users",
submit_title: "Save Permissions",
redirect: "user",
@@ -55,7 +56,7 @@ export const getUserPreferencesModel = () => {
title: _l("Manage API Key"),
id: "edit-preferences-api-key",
description: _l("Access your current API key or create a new one."),
url: `api/users/${Galaxy.user.id}/api_key/inputs`,
url: `api/users/${user_id}/api_key/inputs`,
icon: "fa-key",
submit_title: "Create a new Key",
submit_icon: "fa-check",
@@ -73,7 +74,7 @@ export const getUserPreferencesModel = () => {
title: _l("Manage Toolbox Filters"),
id: "edit-preferences-toolbox-filters",
description: _l("Customize your Toolbox by displaying or omitting sets of Tools."),
url: `api/users/${Galaxy.user.id}/toolbox_filters/inputs`,
url: `api/users/${user_id}/toolbox_filters/inputs`,
icon: "fa-filter",
submit_title: "Save Filters",
redirect: "user",
client/src/entry/analysis/AnalysisRouter.js
+2 -4
View File
@@ -138,10 +138,8 @@ export const getAnalysisRouter = (Galaxy) => {
});
},
show_user_form: function (form_id) {
const Galaxy = getGalaxyInstance();
const model = getUserPreferencesModel();
model.user_id = Galaxy.params.id;
show_user_form: function (form_id, params) {
const model = getUserPreferencesModel(params.id);
this.page.display(new FormWrapper.View(_.extend(model[form_id], { active_tab: "user" })));
},
lib/galaxy/jobs/handler.py
+5
View File
@@ -21,6 +21,7 @@ from sqlalchemy.sql.expression import (
)
from galaxy import model
from galaxy.exceptions import ObjectNotFound
from galaxy.jobs import (
JobDestination,
JobWrapper,
@@ -1087,6 +1088,10 @@ class DefaultJobDispatcher:
except KeyError:
log.error(f'recover(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}')
job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE)
except ObjectNotFound:
msg = "Could not recover job working directory after Galaxy restart"
log.exception(f"recover(): ({job_wrapper.job_id}) {msg}")
job_wrapper.fail(msg)
def shutdown(self):
failures = []
lib/galaxy/model/__init__.py
+3 -1
View File
@@ -4251,6 +4251,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
dce = alias(DatasetCollectionElement)
depth_collection_type = dataset_collection.collection_type
order_by_columns = [dce.c.element_index]
nesting_level = 0
def attribute_columns(column_collection, attributes, nesting_level=None):
@@ -4269,6 +4270,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
nesting_level += 1
inner_dc = alias(DatasetCollection)
inner_dce = alias(DatasetCollectionElement)
order_by_columns.append(inner_dce.c.element_index)
q = q.join(
inner_dc, inner_dc.c.id == dce.c.child_collection_id
).join(
@@ -4297,7 +4299,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
q = q.add_entity(entity)
if entity == DatasetCollectionElement:
q = q.filter(entity.id == dce.c.id)
return q.distinct()
return q.distinct(*order_by_columns).order_by(*order_by_columns)
@property
def dataset_states_and_extensions_summary(self):
scripts/cleanup_datasets/pgcleanup.py
+124 -1
View File
@@ -12,6 +12,7 @@ import logging
import os
import string
import sys
import time
from collections import namedtuple
from functools import partial
@@ -105,6 +106,7 @@ class Action:
self._debug = app.args.debug
self._update_time = app.args.update_time
self._force_retry = app.args.force_retry
self._epoch_time = str(int(time.time()))
self._days = app.args.days
self._config = app.config
self._update = app._update
@@ -141,6 +143,7 @@ class Action:
self.__log.propagate = False
m = ('==== Log opened: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
self.__log.info(m)
self.__log.info(f'Epoch time for this action: {self._epoch_time}')
def __close_log(self):
m = ('==== Log closed: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
@@ -188,6 +191,7 @@ class Action:
return self._action_sql.format(
update_time_sql=self._update_time_sql,
force_retry_sql=self._force_retry_sql,
epoch_time=self._epoch_time,
)
@property
@@ -334,11 +338,13 @@ class PurgesHDAs:
_purge_hda_dependencies_sql = self._purge_hda_dependencies_sql.format(
update_time_sql=self._update_time_sql,
force_retry_sql=self._force_retry_sql,
epoch_time=self._epoch_time,
)
return self._action_sql.format(
purge_hda_dependencies_sql=_purge_hda_dependencies_sql,
update_time_sql=self._update_time_sql,
force_retry_sql=self._force_retry_sql,
epoch_time=self._epoch_time,
)
@@ -525,6 +531,7 @@ class DeleteInactiveUsers(Action):
class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
"""
- Mark purged all users that are older than the specified number of days.
- Set User.disk_usage = 0 for all users purged in this step.
- Mark purged all Histories whose user_ids are purged in this step.
- Mark purged all HistoryDatasetAssociations whose history_ids are purged in this step.
- Delete all UserGroupAssociations whose user_ids are purged in this step.
@@ -655,6 +662,121 @@ class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
self.log.info('zero_disk_usage user_ids: %s', ' '.join(str(i) for i in user_ids))
class PurgeDeletedUsersGDPR(PurgesHDAs, RemovesMetadataFiles, Action):
"""
- Perform all steps in the PurgeDeletedUsers/purge_deleted_users action
- Obfuscate User.email and User.username for all users purged in this step.
NOTE: Your database must have the pgcrypto extension installed e.g. with:
CREATE EXTENSION IF NOT EXISTS pgcrypto;
"""
_action_sql = """
WITH purged_user_ids
AS ( UPDATE galaxy_user
SET purged = true,
disk_usage = 0,
email = encode(digest(email || '{epoch_time}', 'sha1'), 'hex'),
username = encode(digest(username || '{epoch_time}', 'sha1'), 'hex'){update_time_sql}
WHERE deleted{force_retry_sql}
AND update_time < (NOW() AT TIME ZONE 'utc' - interval '%(days)s days')
RETURNING id),
deleted_uga_ids
AS (DELETE FROM user_group_association
USING purged_user_ids
WHERE user_group_association.user_id = purged_user_ids.id
RETURNING user_group_association.user_id AS user_id,
user_group_association.id AS id),
deleted_ura_ids
AS (DELETE FROM user_role_association
USING role
WHERE role.id = user_role_association.role_id
AND role.type != 'private'
AND user_role_association.user_id IN
(SELECT id
FROM purged_user_ids)
RETURNING user_role_association.user_id AS user_id,
user_role_association.id AS id),
deleted_ua_ids
AS (DELETE FROM user_address
USING purged_user_ids
WHERE user_address.user_id = purged_user_ids.id
RETURNING user_address.user_id AS user_id,
user_address.id AS id),
user_events
AS (INSERT INTO cleanup_event_user_association
(create_time, cleanup_event_id, user_id)
SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
FROM purged_user_ids),
purged_history_ids
AS ( UPDATE history
SET purged = true{update_time_sql}
FROM purged_user_ids
WHERE purged_user_ids.id = history.user_id
AND NOT history.purged
RETURNING history.user_id AS user_id,
history.id AS id),
history_events
AS (INSERT INTO cleanup_event_history_association
(create_time, cleanup_event_id, history_id)
SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
FROM purged_history_ids),
purged_hda_ids
AS ( UPDATE history_dataset_association
SET purged = true, deleted = true{update_time_sql}
FROM purged_history_ids
WHERE purged_history_ids.id = history_dataset_association.history_id
AND NOT history_dataset_association.purged
RETURNING history_dataset_association.history_id AS history_id,
history_dataset_association.id AS id),
hda_events
AS (INSERT INTO cleanup_event_hda_association
(create_time, cleanup_event_id, hda_id)
SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
FROM purged_hda_ids),
{purge_hda_dependencies_sql}
SELECT purged_user_ids.id AS purged_user_id,
purged_user_ids.id AS zero_disk_usage_user_id,
purged_history_ids.id AS purged_history_id,
purged_hda_ids.id AS purged_hda_id,
deleted_metadata_file_ids.id AS deleted_metadata_file_id,
deleted_metadata_file_ids.object_store_id AS object_store_id,
deleted_icda_ids.id AS deleted_icda_id,
deleted_icda_ids.hda_id AS deleted_icda_hda_id,
deleted_uga_ids.id AS deleted_uga_id,
deleted_ura_ids.id AS deleted_ura_id,
deleted_ua_ids.id AS deleted_ua_id
FROM purged_user_ids
LEFT OUTER JOIN purged_history_ids
ON purged_user_ids.id = purged_history_ids.user_id
LEFT OUTER JOIN purged_hda_ids
ON purged_history_ids.id = purged_hda_ids.history_id
LEFT OUTER JOIN deleted_metadata_file_ids
ON deleted_metadata_file_ids.hda_id = purged_hda_ids.id
LEFT OUTER JOIN deleted_icda_ids
ON deleted_icda_ids.hda_parent_id = purged_hda_ids.id
LEFT OUTER JOIN deleted_uga_ids
ON purged_user_ids.id = deleted_uga_ids.user_id
LEFT OUTER JOIN deleted_ura_ids
ON purged_user_ids.id = deleted_ura_ids.user_id
LEFT OUTER JOIN deleted_ua_ids
ON purged_user_ids.id = deleted_ua_ids.user_id
ORDER BY purged_user_ids.id
"""
causals = (
('purged_user_id', 'purged_history_id'),
('purged_history_id', 'purged_hda_id'),
('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
('purged_user_id', 'deleted_uga_id'),
('purged_user_id', 'deleted_ura_id'),
('purged_user_id', 'deleted_ua_id'),
)
@classmethod
def name_c(cls):
return 'purge_deleted_users_gdpr'
class PurgeDeletedHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalculation, Action):
"""
- Mark purged all HistoryDatasetAssociations currently marked deleted that are older than the
@@ -1103,7 +1225,8 @@ class Cleanup:
def _execute(self, sql, args):
cur = self.conn.cursor()
log.debug("SQL is: %s", cur.mogrify(sql, args))
sql_str = cur.mogrify(sql, args).decode('utf-8')
log.debug(f"SQL is: {sql_str}")
log.info("Executing SQL")
cur.execute(sql, args)
log.info('Database status: %s', cur.statusmessage)
test/unit/data/test_galaxy_mapping.py
+37
View File
@@ -1,5 +1,6 @@
import collections
import os
import random
import unittest
import uuid
from tempfile import NamedTemporaryFile
@@ -290,6 +291,42 @@ class MappingTests(BaseModelTestCase):
for i in range(elements):
assert c1[i] == dces[i]
def test_dataset_instance_order(self):
model = self.model
u = model.User(email="mary@example.com", password="password")
h1 = model.History(name="History 1", user=u)
elements = []
list_pair = model.DatasetCollection(collection_type="list:paired")
for i in range(20):
pair = model.DatasetCollection(collection_type="pair")
forward = model.HistoryDatasetAssociation(extension="txt", history=h1, name=f"forward_{i}", create_dataset=True, sa_session=model.session)
reverse = model.HistoryDatasetAssociation(extension="bam", history=h1, name=f"reverse_{i}", create_dataset=True, sa_session=model.session)
dce1 = model.DatasetCollectionElement(collection=pair, element=forward, element_identifier=f"forward_{i}", element_index=1)
dce2 = model.DatasetCollectionElement(collection=pair, element=reverse, element_identifier=f"reverse_{i}", element_index=2)
to_persist = [(forward, reverse), (dce1, dce2)]
self.persist(pair)
for item in to_persist:
if i % 2:
self.persist(item[0])
self.persist(item[1])
else:
self.persist(item[1])
self.persist(item[0])
elements.append(model.DatasetCollectionElement(collection=list_pair, element=pair, element_index=i, element_identifier=str(i)))
self.persist(list_pair)
random.shuffle(elements)
for item in elements:
self.persist(item)
forward = []
reverse = []
for i, dataset_instance in enumerate(list_pair.dataset_instances):
if i % 2:
reverse.append(dataset_instance)
else:
forward.append(dataset_instance)
assert all(d.name == f"forward_{i}" for i, d in enumerate(forward))
assert all(d.name == f"reverse_{i}" for i, d in enumerate(reverse))
def test_collections_in_histories(self):
model = self.model