diff --git a/orion-ops-service/src/main/java/com/orion/ops/consts/Const.java b/orion-ops-service/src/main/java/com/orion/ops/consts/Const.java index 8605e6a6..8b1696be 100644 --- a/orion-ops-service/src/main/java/com/orion/ops/consts/Const.java +++ b/orion-ops-service/src/main/java/com/orion/ops/consts/Const.java @@ -56,6 +56,10 @@ public class Const extends com.orion.constant.Const { public static final String TASK_DIR = "/task"; + public static final String IMPORT_DIR = "/import"; + + public static final String EXPORT_DIR = "/export"; + public static final String BUILD = "build"; public static final String RELEASE = "release"; @@ -126,6 +130,10 @@ public class Const extends com.orion.constant.Const { public static final String REJECT_LABEL = "驳回"; + public static final String SUCCESS_LABEL = "成功"; + + public static final String FAILURE_LABEL = "失败"; + public static final Double D_0 = 0D; public static final int DEFAULT_FILE_CLEAN_THRESHOLD = 60; diff --git a/orion-ops-service/src/main/java/com/orion/ops/consts/KeyConst.java b/orion-ops-service/src/main/java/com/orion/ops/consts/KeyConst.java index d9884404..78956785 100644 --- a/orion-ops-service/src/main/java/com/orion/ops/consts/KeyConst.java +++ b/orion-ops-service/src/main/java/com/orion/ops/consts/KeyConst.java @@ -136,4 +136,16 @@ public class KeyConst { */ public static final int DATA_PROFILE_EXPIRE = 60 * 30; + /** + * 数据导入缓存 key + *

+ * data:import:{userId}:{token} + */ + public static final String DATA_IMPORT_TOKEN = "data:import:{}:{}"; + + /** + * 5 min + */ + public static final int DATA_IMPORT_TOKEN_EXPIRE = 60 * 5; + } diff --git a/orion-ops-service/src/main/java/com/orion/ops/consts/MessageConst.java b/orion-ops-service/src/main/java/com/orion/ops/consts/MessageConst.java index 40acdda2..84094d8e 100644 --- a/orion-ops-service/src/main/java/com/orion/ops/consts/MessageConst.java +++ b/orion-ops-service/src/main/java/com/orion/ops/consts/MessageConst.java @@ -30,6 +30,12 @@ public class MessageConst { public static final String OPERATOR_ERROR = "操作失败"; + public static final String NETWORK_FLUCTUATION = "当前环境网路波动"; + + public static final String OPEN_TEMPLATE_ERROR = "模板解析失败 请检查模板和密码"; + + public static final String PARSE_TEMPLATE_DATA_ERROR = "模板解析失败 请检查模板数据"; + public static final String VCS_OPERATOR_ERROR = "版本控制仓库操作执行失败"; public static final String TASK_ERROR = "任务执行异常"; @@ -208,4 +214,6 @@ public class MessageConst { public static final String SFTP_ZIP_FILE_ABSENT = "压缩文件不存在"; + public static final String OPERATOR_TIMEOUT = "操作超时"; + } diff --git a/orion-ops-service/src/main/java/com/orion/ops/consts/SchedulerPools.java b/orion-ops-service/src/main/java/com/orion/ops/consts/SchedulerPools.java index f9be3d86..a3af733c 100644 --- a/orion-ops-service/src/main/java/com/orion/ops/consts/SchedulerPools.java +++ b/orion-ops-service/src/main/java/com/orion/ops/consts/SchedulerPools.java @@ -176,6 +176,18 @@ public class SchedulerPools { .setAllowCoreThreadTimeOut(true) .build(); + /** + * 异步导入 线程池 + */ + public static final ThreadPoolExecutor ASYNC_IMPORT_SCHEDULER = ExecutorBuilder.create() + .setNamedThreadFactory("async-import-thread-") + .setCorePoolSize(1) + .setMaxPoolSize(4) + .setKeepAliveTime(Const.MS_S_30) + .setWorkQueue(new LinkedBlockingQueue<>()) + .setAllowCoreThreadTimeOut(true) + .build(); + static { Systems.addShutdownHook(() -> { Threads.shutdownPoolNow(TERMINAL_SCHEDULER, Const.MS_S_3); @@ -191,6 +203,7 @@ public class SchedulerPools { Threads.shutdownPoolNow(SCHEDULER_TASK_MAIN_SCHEDULER, Const.MS_S_3); Threads.shutdownPoolNow(SCHEDULER_TASK_MACHINE_SCHEDULER, Const.MS_S_3); Threads.shutdownPoolNow(PIPELINE_SCHEDULER, Const.MS_S_3); + Threads.shutdownPoolNow(ASYNC_IMPORT_SCHEDULER, Const.MS_S_3); }); } diff --git a/orion-ops-service/src/main/java/com/orion/ops/consts/event/EventClassify.java b/orion-ops-service/src/main/java/com/orion/ops/consts/event/EventClassify.java index 70268e50..27d3d0ce 100644 --- a/orion-ops-service/src/main/java/com/orion/ops/consts/event/EventClassify.java +++ b/orion-ops-service/src/main/java/com/orion/ops/consts/event/EventClassify.java @@ -114,16 +114,26 @@ public enum EventClassify { */ SYSTEM(95), - /** - * 数据清理 - */ - DATA_CLEAR(97), - /** * 调度操作 */ SCHEDULER(100), + /** + * 数据清理 + */ + DATA_CLEAR(110), + + /** + * 数据导入 + */ + DATA_IMPORT(120), + + /** + * 数据导出 + */ + DATA_EXPORT(130), + ; /** diff --git a/orion-ops-service/src/main/java/com/orion/ops/consts/event/EventType.java b/orion-ops-service/src/main/java/com/orion/ops/consts/event/EventType.java index 6d73e118..a5cc24c7 100644 --- a/orion-ops-service/src/main/java/com/orion/ops/consts/event/EventType.java +++ b/orion-ops-service/src/main/java/com/orion/ops/consts/event/EventType.java @@ -574,49 +574,6 @@ public enum EventType { */ UPDATE_SYSTEM_OPTION(6120, EventClassify.SYSTEM, "修改系统配置项 ${label} 原始值:${before} 修改为:${after}"), - - // -------------------- 数据清理 -------------------- - - /** - * 清理 批量执行数据 - */ - DATA_CLEAR_BATCH_EXEC(6205, EventClassify.DATA_CLEAR, "清理批量执行数据 ${count}条"), - - /** - * 清理 终端日志 - */ - DATA_CLEAR_TERMINAL_LOG(6210, EventClassify.DATA_CLEAR, "清理终端日志 ${count}条"), - - /** - * 清理 调度记录 - */ - DATA_CLEAR_SCHEDULER_RECORD(6215, EventClassify.DATA_CLEAR, "清理任务调度记录 ${count}条"), - - /** - * 清理 应用构建 - */ - DATA_CLEAR_APP_BUILD(6220, EventClassify.DATA_CLEAR, "清理应用构建任务数据 ${count}条"), - - /** - * 清理 应用发布 - */ - DATA_CLEAR_APP_RELEASE(6225, EventClassify.DATA_CLEAR, "清理应用发布任务数据 ${count}条"), - - /** - * 清理 应用流水线 - */ - DATA_CLEAR_APP_PIPELINE(6230, EventClassify.DATA_CLEAR, "清理应用流水线执行数据 ${count}条"), - - /** - * 清理 站内信 - */ - DATA_CLEAR_WEB_SIDE_MESSAGE(6235, EventClassify.DATA_CLEAR, "清理站内信 ${count}条"), - - /** - * 清理 操作日志 - */ - DATA_CLEAR_EVENT_LOG(6240, EventClassify.DATA_CLEAR, "清理操作日志 ${count}条"), - // -------------------- 调度操作 -------------------- /** @@ -664,6 +621,62 @@ public enum EventType { */ DELETE_TASK_RECORD(7145, EventClassify.SCHEDULER, "删除调度任务明细 ${count}个"), + // -------------------- 数据清理 -------------------- + + /** + * 清理 批量执行数据 + */ + DATA_CLEAR_BATCH_EXEC(8105, EventClassify.DATA_CLEAR, "清理批量执行数据 ${count}条"), + + /** + * 清理 终端日志 + */ + DATA_CLEAR_TERMINAL_LOG(8110, EventClassify.DATA_CLEAR, "清理终端日志 ${count}条"), + + /** + * 清理 调度记录 + */ + DATA_CLEAR_SCHEDULER_RECORD(8115, EventClassify.DATA_CLEAR, "清理任务调度记录 ${count}条"), + + /** + * 清理 应用构建 + */ + DATA_CLEAR_APP_BUILD(8120, EventClassify.DATA_CLEAR, "清理应用构建任务数据 ${count}条"), + + /** + * 清理 应用发布 + */ + DATA_CLEAR_APP_RELEASE(8125, EventClassify.DATA_CLEAR, "清理应用发布任务数据 ${count}条"), + + /** + * 清理 应用流水线 + */ + DATA_CLEAR_APP_PIPELINE(8130, EventClassify.DATA_CLEAR, "清理应用流水线执行数据 ${count}条"), + + /** + * 清理 站内信 + */ + DATA_CLEAR_WEB_SIDE_MESSAGE(8135, EventClassify.DATA_CLEAR, "清理站内信 ${count}条"), + + /** + * 清理 操作日志 + */ + DATA_CLEAR_EVENT_LOG(8140, EventClassify.DATA_CLEAR, "清理操作日志 ${count}条"), + + // -------------------- 数据导入 -------------------- + + /** + * 导入机器 + */ + DATA_IMPORT_MACHINE(8205, EventClassify.DATA_IMPORT, "批量导入机器信息"), + + // -------------------- 数据导出 -------------------- + + /** + * 导出机器 + */ + DATA_EXPORT_MACHINE(8305, EventClassify.DATA_EXPORT, "批量导出机器信息"), + ; /** diff --git a/orion-ops-service/src/main/java/com/orion/ops/consts/machine/MachineFieldConst.java b/orion-ops-service/src/main/java/com/orion/ops/consts/machine/MachineFieldConst.java new file mode 100644 index 00000000..e341c0af --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/consts/machine/MachineFieldConst.java @@ -0,0 +1,110 @@ +package com.orion.ops.consts.machine; + +import com.orion.ops.entity.dto.MachineInfoImportDTO; +import com.orion.ops.utils.Valid; +import com.orion.ops.utils.ValueMix; +import com.orion.utils.Strings; + +/** + * 机器表字段常量 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/27 13:49 + */ +public class MachineFieldConst { + + private MachineFieldConst() { + } + + public static final int NAME_MAX_LEN = 32; + + public static final int TAG_MAX_LEN = 32; + + public static final int HOST_MAX_LEN = 128; + + public static final int USER_NAME_MAX_LEN = 32; + + public static final int PASSWORD_MAX_LEN = 128; + + public static final int PORT_MAX_RANGE = 2; + + public static final int PORT_MIN_RANGE = 65534; + + public static final int DESCRIPTION_MAX_LEN = 64; + + public static final String NAME_EMPTY_MESSAGE = "名称不能为空"; + + public static final String NAME_LEN_MESSAGE = "名称长度不能大于 " + NAME_MAX_LEN + "位"; + + public static final String TAG_EMPTY_MESSAGE = "唯一标识不能为空"; + + public static final String TAG_LEN_MESSAGE = "唯一标识长度不能大于 " + TAG_MAX_LEN + "位"; + + public static final String HOST_EMPTY_MESSAGE = "主机不能为空"; + + public static final String HOST_LEN_MESSAGE = "主机长度不能大于 " + HOST_MAX_LEN + "位"; + + public static final String USER_NAME_EMPTY_MESSAGE = "用户名不能为空"; + + public static final String USER_NAME_LEN_MESSAGE = "用户名长度不能大于 " + USER_NAME_MAX_LEN + "位"; + + public static final String PASSWORD_DECRYPT_MESSAGE = "密码密文解密失败"; + + public static final String PASSWORD_LEN_MESSAGE = "密码明文长度不能大于 " + PASSWORD_MAX_LEN + "位"; + + public static final String PORT_EMPTY_MESSAGE = "端口不能为空"; + + public static final String PORT_LEN_MESSAGE = "端口必须在" + PORT_MAX_RANGE + "~" + PORT_MIN_RANGE + "之间"; + + public static final String DESCRIPTION_LEN_MESSAGE = "描述长度不能大于 " + DESCRIPTION_MAX_LEN + "位"; + + + /** + * 验证数据合法性 + * + * @param o o + */ + public static void validData(Object o) { + if (o instanceof MachineInfoImportDTO) { + validImport((MachineInfoImportDTO) o); + } + } + + /** + * 验证导入数据 + * + * @param row row + */ + public static void validImport(MachineInfoImportDTO row) { + String name = row.getName(); + String tag = row.getTag(); + String host = row.getHost(); + String username = row.getUsername(); + Integer port = row.getPort(); + String encryptPassword = row.getEncryptPassword(); + String password = row.getImportPassword(); + String description = row.getDescription(); + Valid.notBlank(name, NAME_EMPTY_MESSAGE); + Valid.validLengthLte(name, NAME_MAX_LEN, NAME_LEN_MESSAGE); + Valid.notBlank(tag, TAG_EMPTY_MESSAGE); + Valid.validLengthLte(tag, TAG_MAX_LEN, TAG_LEN_MESSAGE); + Valid.notBlank(host, HOST_EMPTY_MESSAGE); + Valid.validLengthLte(host, HOST_MAX_LEN, HOST_LEN_MESSAGE); + Valid.notBlank(username, USER_NAME_EMPTY_MESSAGE); + Valid.validLengthLte(username, USER_NAME_MAX_LEN, USER_NAME_LEN_MESSAGE); + Valid.notNull(port, PORT_EMPTY_MESSAGE); + Valid.inRange(port, PORT_MAX_RANGE, PORT_MIN_RANGE, PORT_LEN_MESSAGE); + if (!Strings.isBlank(encryptPassword)) { + String decryptPassword = ValueMix.decrypt(encryptPassword); + Valid.notNull(decryptPassword, PASSWORD_DECRYPT_MESSAGE); + row.setDecryptPassword(decryptPassword); + } + if (!Strings.isBlank(password)) { + Valid.validLengthLte(password, PASSWORD_MAX_LEN, PASSWORD_LEN_MESSAGE); + } + if (!Strings.isBlank(description)) { + Valid.validLengthLte(description, DESCRIPTION_MAX_LEN, DESCRIPTION_LEN_MESSAGE); + } + } +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/entity/dto/MachineInfoImportDTO.java b/orion-ops-service/src/main/java/com/orion/ops/entity/dto/MachineInfoImportDTO.java new file mode 100644 index 00000000..3ad958e0 --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/entity/dto/MachineInfoImportDTO.java @@ -0,0 +1,107 @@ +package com.orion.ops.entity.dto; + +import com.orion.office.excel.annotation.ImportField; +import com.orion.ops.consts.machine.MachineAuthType; +import com.orion.ops.entity.domain.MachineInfoDO; +import com.orion.ops.entity.vo.DataImportCheckRowVO; +import com.orion.ops.utils.ValueMix; +import com.orion.utils.Strings; +import com.orion.utils.convert.TypeStore; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * 机器导入数据 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/26 14:01 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class MachineInfoImportDTO extends BaseDataImportDTO { + + /** + * 机器名称 + */ + @ImportField(index = 0) + private String name; + + /** + * 机器标签 + */ + @ImportField(index = 1) + private String tag; + + /** + * 机器主机 + */ + @ImportField(index = 2) + private String host; + + /** + * ssh 端口 + */ + @ImportField(index = 3) + private Integer port; + + /** + * 用户名 + */ + @ImportField(index = 4) + private String username; + + /** + * 密码 (密文) + */ + @ImportField(index = 5) + private String encryptPassword; + + /** + * 导入密码 (明文) + */ + @ImportField(index = 6) + private String importPassword; + + /** + * 描述 + */ + @ImportField(index = 7) + private String description; + + /** + * 密码密文解密后的明文 + */ + private String decryptPassword; + + static { + TypeStore.STORE.register(MachineInfoImportDTO.class, DataImportCheckRowVO.class, p -> { + DataImportCheckRowVO vo = new DataImportCheckRowVO(); + vo.setSymbol(p.getTag()); + vo.setIllegalMessage(p.getIllegalMessage()); + return vo; + }); + TypeStore.STORE.register(MachineInfoImportDTO.class, MachineInfoDO.class, p -> { + MachineInfoDO d = new MachineInfoDO(); + d.setMachineName(p.getName()); + d.setMachineTag(p.getTag()); + d.setMachineHost(p.getHost()); + d.setSshPort(p.getPort()); + d.setDescription(p.getDescription()); + d.setUsername(p.getUsername()); + if (!Strings.isBlank(p.getDecryptPassword())) { + d.setPassword(ValueMix.encrypt(p.getDecryptPassword())); + } + if (!Strings.isBlank(p.getImportPassword())) { + d.setPassword(ValueMix.encrypt(p.getImportPassword())); + } + if (d.getPassword() == null) { + d.setAuthType(MachineAuthType.KEY.getType()); + } else { + d.setAuthType(MachineAuthType.PASSWORD.getType()); + } + return d; + }); + } + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/runner/CacheKeyCleanRunner.java b/orion-ops-service/src/main/java/com/orion/ops/runner/CacheKeyCleanRunner.java index 57341559..07bc947f 100644 --- a/orion-ops-service/src/main/java/com/orion/ops/runner/CacheKeyCleanRunner.java +++ b/orion-ops-service/src/main/java/com/orion/ops/runner/CacheKeyCleanRunner.java @@ -52,7 +52,9 @@ public class CacheKeyCleanRunner implements CommandLineRunner { // 调度统计key Strings.format(KeyConst.SCHEDULER_TASK_STATISTICS_KEY, "*"), // 环境缓存key - KeyConst.DATA_PROFILE_KEY + KeyConst.DATA_PROFILE_KEY, + // 数据导入缓存 + Strings.format(KeyConst.DATA_IMPORT_TOKEN, "*", "*") ); // 查询删除缓存key scanKeys.stream()