opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 13:50:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release_21.05' into dev

Browse Source
This commit is contained in:
mvdbeek
2021-08-09 18:45:05 +02:00
parent 02e570d722 b38b41a1e4
commit ecc54ba00c
7 changed files with 179 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cleanup_datasets/pgcleanup.py
+124 -1
View File
@@ -12,6 +12,7 @@ import logging
import os
import string
import sys
import time
from collections import namedtuple
from functools import partial
@@ -105,6 +106,7 @@ class Action:
self._debug = app.args.debug
self._update_time = app.args.update_time
self._force_retry = app.args.force_retry
self._epoch_time = str(int(time.time()))
self._days = app.args.days
self._config = app.config
self._update = app._update
@@ -141,6 +143,7 @@ class Action:
self.__log.propagate = False
m = ('==== Log opened: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
self.__log.info(m)
self.__log.info(f'Epoch time for this action: {self._epoch_time}')
def __close_log(self):
m = ('==== Log closed: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
@@ -188,6 +191,7 @@ class Action:
return self._action_sql.format(
update_time_sql=self._update_time_sql,
force_retry_sql=self._force_retry_sql,
epoch_time=self._epoch_time,
)
@property
@@ -334,11 +338,13 @@ class PurgesHDAs:
_purge_hda_dependencies_sql = self._purge_hda_dependencies_sql.format(
update_time_sql=self._update_time_sql,
force_retry_sql=self._force_retry_sql,
epoch_time=self._epoch_time,
)
return self._action_sql.format(
purge_hda_dependencies_sql=_purge_hda_dependencies_sql,
update_time_sql=self._update_time_sql,
force_retry_sql=self._force_retry_sql,
epoch_time=self._epoch_time,
)
@@ -525,6 +531,7 @@ class DeleteInactiveUsers(Action):
class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
"""
- Mark purged all users that are older than the specified number of days.
- Set User.disk_usage = 0 for all users purged in this step.
- Mark purged all Histories whose user_ids are purged in this step.
- Mark purged all HistoryDatasetAssociations whose history_ids are purged in this step.
- Delete all UserGroupAssociations whose user_ids are purged in this step.
@@ -655,6 +662,121 @@ class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
self.log.info('zero_disk_usage user_ids: %s', ' '.join(str(i) for i in user_ids))
class PurgeDeletedUsersGDPR(PurgesHDAs, RemovesMetadataFiles, Action):
"""
- Perform all steps in the PurgeDeletedUsers/purge_deleted_users action
- Obfuscate User.email and User.username for all users purged in this step.
NOTE: Your database must have the pgcrypto extension installed e.g. with:
CREATE EXTENSION IF NOT EXISTS pgcrypto;
"""
_action_sql = """
WITH purged_user_ids
AS ( UPDATE galaxy_user
SET purged = true,
disk_usage = 0,
email = encode(digest(email || '{epoch_time}', 'sha1'), 'hex'),
username = encode(digest(username || '{epoch_time}', 'sha1'), 'hex'){update_time_sql}
WHERE deleted{force_retry_sql}
AND update_time < (NOW() AT TIME ZONE 'utc' - interval '%(days)s days')
RETURNING id),
deleted_uga_ids
AS (DELETE FROM user_group_association
USING purged_user_ids
WHERE user_group_association.user_id = purged_user_ids.id
RETURNING user_group_association.user_id AS user_id,
user_group_association.id AS id),
deleted_ura_ids
AS (DELETE FROM user_role_association
USING role
WHERE role.id = user_role_association.role_id
AND role.type != 'private'
AND user_role_association.user_id IN
(SELECT id
FROM purged_user_ids)
RETURNING user_role_association.user_id AS user_id,
user_role_association.id AS id),
deleted_ua_ids
AS (DELETE FROM user_address
USING purged_user_ids
WHERE user_address.user_id = purged_user_ids.id
RETURNING user_address.user_id AS user_id,
user_address.id AS id),
user_events
AS (INSERT INTO cleanup_event_user_association
(create_time, cleanup_event_id, user_id)
SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
FROM purged_user_ids),
purged_history_ids
AS ( UPDATE history
SET purged = true{update_time_sql}
FROM purged_user_ids
WHERE purged_user_ids.id = history.user_id
AND NOT history.purged
RETURNING history.user_id AS user_id,
history.id AS id),
history_events
AS (INSERT INTO cleanup_event_history_association
(create_time, cleanup_event_id, history_id)
SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
FROM purged_history_ids),
purged_hda_ids
AS ( UPDATE history_dataset_association
SET purged = true, deleted = true{update_time_sql}
FROM purged_history_ids
WHERE purged_history_ids.id = history_dataset_association.history_id
AND NOT history_dataset_association.purged
RETURNING history_dataset_association.history_id AS history_id,
history_dataset_association.id AS id),
hda_events
AS (INSERT INTO cleanup_event_hda_association
(create_time, cleanup_event_id, hda_id)
SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
FROM purged_hda_ids),
{purge_hda_dependencies_sql}
SELECT purged_user_ids.id AS purged_user_id,
purged_user_ids.id AS zero_disk_usage_user_id,
purged_history_ids.id AS purged_history_id,
purged_hda_ids.id AS purged_hda_id,
deleted_metadata_file_ids.id AS deleted_metadata_file_id,
deleted_metadata_file_ids.object_store_id AS object_store_id,
deleted_icda_ids.id AS deleted_icda_id,
deleted_icda_ids.hda_id AS deleted_icda_hda_id,
deleted_uga_ids.id AS deleted_uga_id,
deleted_ura_ids.id AS deleted_ura_id,
deleted_ua_ids.id AS deleted_ua_id
FROM purged_user_ids
LEFT OUTER JOIN purged_history_ids
ON purged_user_ids.id = purged_history_ids.user_id
LEFT OUTER JOIN purged_hda_ids
ON purged_history_ids.id = purged_hda_ids.history_id
LEFT OUTER JOIN deleted_metadata_file_ids
ON deleted_metadata_file_ids.hda_id = purged_hda_ids.id
LEFT OUTER JOIN deleted_icda_ids
ON deleted_icda_ids.hda_parent_id = purged_hda_ids.id
LEFT OUTER JOIN deleted_uga_ids
ON purged_user_ids.id = deleted_uga_ids.user_id
LEFT OUTER JOIN deleted_ura_ids
ON purged_user_ids.id = deleted_ura_ids.user_id
LEFT OUTER JOIN deleted_ua_ids
ON purged_user_ids.id = deleted_ua_ids.user_id
ORDER BY purged_user_ids.id
"""
causals = (
('purged_user_id', 'purged_history_id'),
('purged_history_id', 'purged_hda_id'),
('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
('purged_user_id', 'deleted_uga_id'),
('purged_user_id', 'deleted_ura_id'),
('purged_user_id', 'deleted_ua_id'),
)
@classmethod
def name_c(cls):
return 'purge_deleted_users_gdpr'
class PurgeDeletedHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalculation, Action):
"""
- Mark purged all HistoryDatasetAssociations currently marked deleted that are older than the
@@ -1103,7 +1225,8 @@ class Cleanup:
def _execute(self, sql, args):
cur = self.conn.cursor()
log.debug("SQL is: %s", cur.mogrify(sql, args))
sql_str = cur.mogrify(sql, args).decode('utf-8')
log.debug(f"SQL is: {sql_str}")
log.info("Executing SQL")
cur.execute(sql, args)
log.info('Database status: %s', cur.statusmessage)