feat: add batch clear data api.

This commit is contained in:
lijiahang
2022-05-26 08:38:12 +08:00
parent 9ecfebc4c2
commit 1bbfd22a1a
11 changed files with 714 additions and 5 deletions
@@ -0,0 +1,48 @@
package com.orion.ops.consts.clear;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* 数据清理区间
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/24 16:29
*/
@Getter
@AllArgsConstructor
public enum DataClearRange {
/**
* 保留天数
*/
DAY(10),
/**
* 保留条数
*/
TOTAL(20),
/**
* 关联数据
*/
REL_ID(30),
;
private final Integer range;
public static DataClearRange of(Integer range) {
if (range == null) {
return null;
}
for (DataClearRange value : values()) {
if (value.range.equals(range)) {
return value;
}
}
return null;
}
}
@@ -114,6 +114,11 @@ public enum EventClassify {
*/
SYSTEM(95),
/**
* 数据清理
*/
DATA_CLEAR(97),
/**
* 调度操作
*/
@@ -574,6 +574,49 @@ public enum EventType {
*/
UPDATE_SYSTEM_OPTION(6120, EventClassify.SYSTEM, "修改系统配置项 <sb>${label}</sb> 原始值:<sb>${before}</sb> 修改为:<sb>${after}</sb>"),
// -------------------- 数据清理 --------------------
/**
* 清理 批量执行数据
*/
DATA_CLEAR_BATCH_EXEC(6205, EventClassify.DATA_CLEAR, "清理批量执行数据 <sb>${count}</sb>条"),
/**
* 清理 终端日志
*/
DATA_CLEAR_TERMINAL_LOG(6210, EventClassify.DATA_CLEAR, "清理终端日志 <sb>${count}</sb>条"),
/**
* 清理 调度记录
*/
DATA_CLEAR_SCHEDULER_RECORD(6215, EventClassify.DATA_CLEAR, "清理任务调度记录 <sb>${count}</sb>条"),
/**
* 清理 应用构建
*/
DATA_CLEAR_APP_BUILD(6220, EventClassify.DATA_CLEAR, "清理应用构建任务数据 <sb>${count}</sb>条"),
/**
* 清理 应用发布
*/
DATA_CLEAR_APP_RELEASE(6225, EventClassify.DATA_CLEAR, "清理应用发布任务数据 <sb>${count}</sb>条"),
/**
* 清理 应用流水线
*/
DATA_CLEAR_APP_PIPELINE(6230, EventClassify.DATA_CLEAR, "清理应用流水线执行数据 <sb>${count}</sb>条"),
/**
* 清理 站内信
*/
DATA_CLEAR_WEB_SIDE_MESSAGE(6235, EventClassify.DATA_CLEAR, "清理站内信 <sb>${count}</sb>条"),
/**
* 清理 操作日志
*/
DATA_CLEAR_EVENT_LOG(6240, EventClassify.DATA_CLEAR, "清理操作日志 <sb>${count}</sb>条"),
// -------------------- 调度操作 --------------------
/**
@@ -0,0 +1,144 @@
package com.orion.ops.controller;
import com.orion.ops.annotation.EventLog;
import com.orion.ops.annotation.RequireRole;
import com.orion.ops.annotation.RestWrapper;
import com.orion.ops.consts.Const;
import com.orion.ops.consts.clear.DataClearRange;
import com.orion.ops.consts.event.EventType;
import com.orion.ops.consts.user.RoleType;
import com.orion.ops.entity.request.sftp.DataClearRequest;
import com.orion.ops.service.api.DataClearService;
import com.orion.ops.utils.Currents;
import com.orion.ops.utils.Valid;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
/**
* 数据清理
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/24 16:50
*/
@RestController
@RestWrapper
@RequestMapping("/orion/api/data-clear")
public class DataClearController {
@Resource
private DataClearService dataClearService;
/**
* 清理 批量执行
*/
@RequestMapping("/batch-exec")
@EventLog(EventType.DATA_CLEAR_BATCH_EXEC)
public Integer clearBatchExec(@RequestBody DataClearRequest request) {
this.validParams(request);
if (!Currents.isAdministrator()) {
request.setICreated(Const.ENABLE);
}
return dataClearService.clearBatchExec(request);
}
/**
* 清理 终端日志
*/
@RequestMapping("/terminal-log")
@RequireRole(RoleType.ADMINISTRATOR)
@EventLog(EventType.DATA_CLEAR_TERMINAL_LOG)
public Integer clearTerminalLog(@RequestBody DataClearRequest request) {
this.validParams(request);
return dataClearService.clearTerminalLog(request);
}
/**
* 清理 调度记录
*/
@RequestMapping("/scheduler-record")
@EventLog(EventType.DATA_CLEAR_SCHEDULER_RECORD)
public Integer clearSchedulerRecord(@RequestBody DataClearRequest request) {
this.validParams(request);
return dataClearService.clearSchedulerRecord(request);
}
/**
* 清理 应用构建
*/
@RequestMapping("/app-build")
@EventLog(EventType.DATA_CLEAR_APP_BUILD)
public Integer clearAppBuild(@RequestBody DataClearRequest request) {
this.validParams(request);
Valid.notNull(request.getProfileId());
return dataClearService.clearAppBuild(request);
}
/**
* 清理 应用发布
*/
@RequestMapping("/app-release")
@EventLog(EventType.DATA_CLEAR_APP_RELEASE)
public Integer clearAppRelease(@RequestBody DataClearRequest request) {
this.validParams(request);
Valid.notNull(request.getProfileId());
return dataClearService.clearAppRelease(request);
}
/**
* 清理 应用流水线
*/
@RequestMapping("/app-pipeline")
@EventLog(EventType.DATA_CLEAR_APP_PIPELINE)
public Integer clearAppPipeline(@RequestBody DataClearRequest request) {
this.validParams(request);
Valid.notNull(request.getProfileId());
return dataClearService.clearAppPipeline(request);
}
/**
* 清理 站内信
*/
@RequestMapping("/web-side-message")
@EventLog(EventType.DATA_CLEAR_WEB_SIDE_MESSAGE)
public Integer clearWebSideMessage(@RequestBody DataClearRequest request) {
this.validParams(request);
return dataClearService.clearWebSideMessage(request);
}
/**
* 清理 操作日志
*/
@RequestMapping("/event-log")
@RequireRole(RoleType.ADMINISTRATOR)
@EventLog(EventType.DATA_CLEAR_EVENT_LOG)
public Integer clearEventLog(@RequestBody DataClearRequest request) {
this.validParams(request);
return dataClearService.clearEventLog(request);
}
/**
* 验证参数
*
* @param request request
*/
private void validParams(DataClearRequest request) {
DataClearRange range = Valid.notNull(DataClearRange.of(request.getRange()));
switch (range) {
case DAY:
Valid.gte(request.getReserveDay(), 0);
return;
case TOTAL:
Valid.gte(request.getReserveTotal(), 0);
return;
case REL_ID:
Valid.notEmpty(request.getRelIdList());
return;
default:
}
}
}
@@ -1,9 +1,6 @@
package com.orion.ops.entity.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import lombok.EqualsAndHashCode;
@@ -80,6 +77,15 @@ public class UserEventLogDO implements Serializable {
@TableField("exec_result")
private Integer execResult;
/**
* 是否删除 1未删除 2已删除
*
* @see com.orion.ops.consts.Const#NOT_DELETED
* @see com.orion.ops.consts.Const#IS_DELETED
*/
@TableLogic
private Integer deleted;
/**
* 创建时间
*/
@@ -0,0 +1,81 @@
package com.orion.ops.entity.request.sftp;
import lombok.Data;
import java.util.List;
/**
* 数据清理请求
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/24 16:36
*/
@Data
public class DataClearRequest {
/**
* 保留天数
*/
private Integer reserveDay;
/**
* 保留条数
*/
private Integer reserveTotal;
/**
* 清理区间
*
* @see com.orion.ops.consts.clear.DataClearRange
*/
private Integer range;
/**
* 清理的引用id
*/
private List<Long> relIdList;
/**
* 引用id
*/
private Long relId;
/**
* profile id
*/
private Long profileId;
/**
* 只清理我创建的
*
* @see com.orion.ops.consts.Const#ENABLE
* @see com.orion.ops.consts.Const#DISABLE
*/
private Integer iCreated;
/**
* 只清理我审核的
*
* @see com.orion.ops.consts.Const#ENABLE
* @see com.orion.ops.consts.Const#DISABLE
*/
private Integer iAudited;
/**
* 只清理我执行的
*
* @see com.orion.ops.consts.Const#ENABLE
* @see com.orion.ops.consts.Const#DISABLE
*/
private Integer iExecute;
/**
* 只清理未读的
*
* @see com.orion.ops.consts.Const#ENABLE
* @see com.orion.ops.consts.Const#DISABLE
*/
private Integer onlyRead;
}
@@ -58,6 +58,7 @@ public class ScpTransferActionHandler extends AbstractActionHandler {
String scpCommand = Strings.def(action.getActionCommand(), CommandConst.SCP_TRANSFER_DEFAULT);
Map<String, String> params = Maps.newMap();
params.put(EnvConst.BUNDLE_PATH, bundlePath);
// 目标文件有空格需要转义空格为 \\
params.put(EnvConst.TRANSFER_PATH, store.getTransferPath().replaceAll(Strings.SPACE, "\\\\\\\\ "));
params.put(EnvConst.TARGET_USERNAME, store.getMachineUsername());
params.put(EnvConst.TARGET_HOST, store.getMachineHost());
@@ -0,0 +1,78 @@
package com.orion.ops.service.api;
import com.orion.ops.entity.request.sftp.DataClearRequest;
/**
* 数据清理服务
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/24 17:01
*/
public interface DataClearService {
/**
* 清理 批量执行数据
*
* @param request request
* @return 删除数量
*/
Integer clearBatchExec(DataClearRequest request);
/**
* 清理 终端日志
*
* @param request request
* @return 删除数量
*/
Integer clearTerminalLog(DataClearRequest request);
/**
* 清理 调度记录
*
* @param request request
* @return 删除数量
*/
Integer clearSchedulerRecord(DataClearRequest request);
/**
* 清理 应用构建
*
* @param request request
* @return 删除数量
*/
Integer clearAppBuild(DataClearRequest request);
/**
* 清理 应用发布
*
* @param request request
* @return 删除数量
*/
Integer clearAppRelease(DataClearRequest request);
/**
* 清理 应用流水线
*
* @param request request
* @return 删除数量
*/
Integer clearAppPipeline(DataClearRequest request);
/**
* 清理 站内信
*
* @param request request
* @return 删除数量
*/
Integer clearWebSideMessage(DataClearRequest request);
/**
* 清理 操作日志
*
* @param request request
* @return 删除数量
*/
Integer clearEventLog(DataClearRequest request);
}
@@ -0,0 +1,290 @@
package com.orion.ops.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.toolkit.support.SFunction;
import com.orion.ops.consts.Const;
import com.orion.ops.consts.app.BuildStatus;
import com.orion.ops.consts.app.PipelineStatus;
import com.orion.ops.consts.app.ReleaseStatus;
import com.orion.ops.consts.clear.DataClearRange;
import com.orion.ops.consts.command.ExecStatus;
import com.orion.ops.consts.event.EventKeys;
import com.orion.ops.consts.event.EventParamsHolder;
import com.orion.ops.consts.message.ReadStatus;
import com.orion.ops.consts.scheduler.SchedulerTaskStatus;
import com.orion.ops.dao.*;
import com.orion.ops.entity.domain.*;
import com.orion.ops.entity.request.sftp.DataClearRequest;
import com.orion.ops.service.api.DataClearService;
import com.orion.ops.utils.Currents;
import com.orion.utils.time.Dates;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Date;
/**
* 数据清理服务实现
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/24 17:04
*/
@Service("dataClearService")
public class DataClearServiceImpl implements DataClearService {
@Resource
private CommandExecDAO commandExecDAO;
@Resource
private MachineTerminalLogDAO machineTerminalLogDAO;
@Resource
private SchedulerTaskRecordDAO schedulerTaskRecordDAO;
@Resource
private ApplicationBuildDAO applicationBuildDAO;
@Resource
private ApplicationReleaseDAO applicationReleaseDAO;
@Resource
private ApplicationPipelineTaskDAO applicationPipelineTaskDAO;
@Resource
private WebSideMessageDAO webSideMessageDAO;
@Resource
private UserEventLogDAO userEventLogDAO;
@Override
public Integer clearBatchExec(DataClearRequest request) {
LambdaQueryWrapper<CommandExecDO> wrapper = new LambdaQueryWrapper<CommandExecDO>()
.ne(CommandExecDO::getExecStatus, ExecStatus.RUNNABLE.getStatus())
.eq(Const.ENABLE.equals(request.getICreated()), CommandExecDO::getUserId, Currents.getUserId());
// 设置删除筛选条件
this.setDeleteWrapper(commandExecDAO, wrapper,
CommandExecDO::getId,
CommandExecDO::getCreateTime,
CommandExecDO::getMachineId,
request);
int count = commandExecDAO.delete(wrapper);
// 设置日志参数
EventParamsHolder.addParams(request);
EventParamsHolder.addParam(EventKeys.COUNT, count);
return count;
}
@Override
public Integer clearTerminalLog(DataClearRequest request) {
LambdaQueryWrapper<MachineTerminalLogDO> wrapper = new LambdaQueryWrapper<>();
// 设置删除筛选条件
this.setDeleteWrapper(machineTerminalLogDAO, wrapper,
MachineTerminalLogDO::getId,
MachineTerminalLogDO::getCreateTime,
MachineTerminalLogDO::getMachineId,
request);
int count = machineTerminalLogDAO.delete(wrapper);
// 设置日志参数
EventParamsHolder.addParams(request);
EventParamsHolder.addParam(EventKeys.COUNT, count);
return count;
}
@Override
public Integer clearSchedulerRecord(DataClearRequest request) {
LambdaQueryWrapper<SchedulerTaskRecordDO> wrapper = new LambdaQueryWrapper<SchedulerTaskRecordDO>()
.ne(SchedulerTaskRecordDO::getTaskStatus, SchedulerTaskStatus.RUNNABLE.getStatus())
.eq(SchedulerTaskRecordDO::getTaskId, request.getRelId());
// 设置删除筛选条件
this.setDeleteWrapper(schedulerTaskRecordDAO, wrapper,
SchedulerTaskRecordDO::getId,
SchedulerTaskRecordDO::getCreateTime,
SchedulerTaskRecordDO::getTaskId,
request);
int count = schedulerTaskRecordDAO.delete(wrapper);
// 设置日志参数
EventParamsHolder.addParams(request);
EventParamsHolder.addParam(EventKeys.COUNT, count);
return count;
}
@Override
public Integer clearAppBuild(DataClearRequest request) {
Long userId = Currents.getUserId();
LambdaQueryWrapper<ApplicationBuildDO> wrapper = new LambdaQueryWrapper<ApplicationBuildDO>()
.ne(ApplicationBuildDO::getBuildStatus, BuildStatus.RUNNABLE.getStatus())
.eq(ApplicationBuildDO::getProfileId, request.getProfileId())
.eq(Const.ENABLE.equals(request.getICreated()), ApplicationBuildDO::getCreateUserId, userId);
// 设置筛选条件
this.setDeleteWrapper(applicationBuildDAO, wrapper,
ApplicationBuildDO::getId,
ApplicationBuildDO::getCreateTime,
ApplicationBuildDO::getAppId,
request);
int count = applicationBuildDAO.delete(wrapper);
// 设置日志参数
EventParamsHolder.addParams(request);
EventParamsHolder.addParam(EventKeys.COUNT, count);
return count;
}
@Override
public Integer clearAppRelease(DataClearRequest request) {
Long userId = Currents.getUserId();
LambdaQueryWrapper<ApplicationReleaseDO> wrapper = new LambdaQueryWrapper<ApplicationReleaseDO>()
.ne(ApplicationReleaseDO::getReleaseStatus, ReleaseStatus.RUNNABLE.getStatus())
.eq(ApplicationReleaseDO::getProfileId, request.getProfileId())
.eq(Const.ENABLE.equals(request.getICreated()), ApplicationReleaseDO::getCreateUserId, userId)
.eq(Const.ENABLE.equals(request.getIAudited()), ApplicationReleaseDO::getAuditUserId, userId)
.eq(Const.ENABLE.equals(request.getIExecute()), ApplicationReleaseDO::getReleaseUserId, userId);
// 设置筛选条件
this.setDeleteWrapper(applicationReleaseDAO, wrapper,
ApplicationReleaseDO::getId,
ApplicationReleaseDO::getCreateTime,
ApplicationReleaseDO::getAppId,
request);
int count = applicationReleaseDAO.delete(wrapper);
// 设置日志参数
EventParamsHolder.addParams(request);
EventParamsHolder.addParam(EventKeys.COUNT, count);
return count;
}
@Override
public Integer clearAppPipeline(DataClearRequest request) {
Long userId = Currents.getUserId();
LambdaQueryWrapper<ApplicationPipelineTaskDO> wrapper = new LambdaQueryWrapper<ApplicationPipelineTaskDO>()
.ne(ApplicationPipelineTaskDO::getExecStatus, PipelineStatus.RUNNABLE.getStatus())
.eq(ApplicationPipelineTaskDO::getProfileId, request.getProfileId())
.eq(Const.ENABLE.equals(request.getICreated()), ApplicationPipelineTaskDO::getCreateUserId, userId)
.eq(Const.ENABLE.equals(request.getIAudited()), ApplicationPipelineTaskDO::getAuditUserId, userId)
.eq(Const.ENABLE.equals(request.getIExecute()), ApplicationPipelineTaskDO::getExecUserId, userId);
// 设置筛选条件
this.setDeleteWrapper(applicationPipelineTaskDAO, wrapper,
ApplicationPipelineTaskDO::getId,
ApplicationPipelineTaskDO::getCreateTime,
ApplicationPipelineTaskDO::getPipelineId,
request);
int count = applicationPipelineTaskDAO.delete(wrapper);
// 设置日志参数
EventParamsHolder.addParams(request);
EventParamsHolder.addParam(EventKeys.COUNT, count);
return count;
}
@Override
public Integer clearWebSideMessage(DataClearRequest request) {
LambdaQueryWrapper<WebSideMessageDO> wrapper = new LambdaQueryWrapper<WebSideMessageDO>()
.eq(WebSideMessageDO::getToUserId, Currents.getUserId())
.eq(Const.ENABLE.equals(request.getOnlyRead()), WebSideMessageDO::getReadStatus, ReadStatus.READ.getStatus());
// 设置删除筛选条件
this.setDeleteWrapper(webSideMessageDAO, wrapper,
WebSideMessageDO::getId,
WebSideMessageDO::getCreateTime,
WebSideMessageDO::getMessageClassify,
request);
int count = webSideMessageDAO.delete(wrapper);
// 设置日志参数
EventParamsHolder.addParams(request);
EventParamsHolder.addParam(EventKeys.COUNT, count);
return count;
}
@Override
public Integer clearEventLog(DataClearRequest request) {
LambdaQueryWrapper<UserEventLogDO> wrapper = new LambdaQueryWrapper<UserEventLogDO>()
.eq(UserEventLogDO::getExecResult, Const.ENABLE);
// 设置删除筛选条件
this.setDeleteWrapper(userEventLogDAO, wrapper,
UserEventLogDO::getId,
UserEventLogDO::getCreateTime,
UserEventLogDO::getEventClassify,
request);
int count = userEventLogDAO.delete(wrapper);
// 设置日志参数
EventParamsHolder.addParams(request);
EventParamsHolder.addParam(EventKeys.COUNT, count);
return count;
}
/**
* 设置删除的筛选条件
*
* @param mapper mapper
* @param wrapper wrapper
* @param idGetterFun idGetterFun
* @param dateGetterFun dateGetterFun
* @param relGetterFun relGetterFun
* @param request request
* @param <T> <T>
*/
private <T> void setDeleteWrapper(BaseMapper<T> mapper,
LambdaQueryWrapper<T> wrapper,
SFunction<T, Long> idGetterFun,
SFunction<T, Date> dateGetterFun,
SFunction<T, ?> relGetterFun,
DataClearRequest request) {
if (DataClearRange.DAY.getRange().equals(request.getRange())) {
// 仅保留几天
Integer day = request.getReserveDay();
if (!day.equals(0)) {
this.setReverseDateWhere(wrapper, dateGetterFun, day);
}
} else if (DataClearRange.TOTAL.getRange().equals(request.getRange())) {
// 保留几条
Integer total = request.getReserveTotal();
if (!total.equals(0)) {
this.setReverseTotalWhere(mapper, wrapper, idGetterFun, total);
}
} else if (DataClearRange.REL_ID.getRange().equals(request.getRange())) {
// 删除机器id
wrapper.in(relGetterFun, request.getRelIdList());
}
}
/**
* 设置保留多少天的筛选条件
*
* @param wrapper wrapper
* @param dateGetterFun dateGetterFun
* @param day day
* @param <T> T
*/
private <T> void setReverseDateWhere(LambdaQueryWrapper<T> wrapper,
SFunction<T, Date> dateGetterFun,
Integer day) {
Date lessDayDate = Dates.stream().subDay(day).get();
wrapper.lt(dateGetterFun, lessDayDate);
}
/**
* 设置保留条数的筛选条件
*
* @param mapper mapper
* @param wrapper wrapper
* @param idGetterFun idGetterFun
* @param total total
* @param <T> T
*/
private <T> void setReverseTotalWhere(BaseMapper<T> mapper,
LambdaQueryWrapper<T> wrapper,
SFunction<T, Long> idGetterFun,
Integer total) {
// 查询删除的最大id
LambdaQueryWrapper<T> maxIdWrapper = wrapper.clone()
.orderByDesc(idGetterFun)
.last(Const.LIMIT + Const.SPACE + total + ", 1");
T maxEntity = mapper.selectOne(maxIdWrapper);
// 未查询到则代表条数不满足, 设置一个 false 条件 从而达到不执行的目的
if (maxEntity == null) {
wrapper.eq(idGetterFun, Const.N_N_L_1);
return;
}
Long maxId = idGetterFun.apply(maxEntity);
// 设置最大id阈值
wrapper.le(idGetterFun, maxId);
}
}
@@ -1,9 +1,12 @@
package com.orion.ops.utils;
import com.orion.lang.wrapper.HttpWrapper;
import com.orion.ops.consts.Const;
import com.orion.ops.consts.ResultCode;
import com.orion.ops.consts.user.RoleType;
import com.orion.ops.consts.user.UserHolder;
import com.orion.ops.entity.dto.UserDTO;
import com.orion.utils.Exceptions;
import javax.servlet.http.HttpServletRequest;
import java.util.Optional;
@@ -88,4 +91,13 @@ public class Currents {
return RoleType.isAdministrator(roleType);
}
/**
* 检查是否为管理员权限
*/
public static void requireAdministrator() {
if (!isAdministrator()) {
throw Exceptions.httpWrapper(HttpWrapper.of(ResultCode.NO_PERMISSION));
}
}
}
@@ -12,12 +12,13 @@
<result column="log_info" property="logInfo"/>
<result column="params_json" property="paramsJson"/>
<result column="exec_result" property="execResult"/>
<result column="deleted" property="deleted"/>
<result column="create_time" property="createTime"/>
</resultMap>
<!-- 通用查询结果列 -->
<sql id="Base_Column_List">
id, user_id, username, event_classify, event_type, log_info, params_json, exec_result, create_time
id, user_id, username, event_classify, event_type, log_info, params_json, exec_result, deleted, create_time
</sql>
</mapper>