diff --git a/orion-ops-service/src/main/java/com/orion/ops/consts/export/ImportType.java b/orion-ops-service/src/main/java/com/orion/ops/consts/export/ImportType.java new file mode 100644 index 00000000..cb103a33 --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/consts/export/ImportType.java @@ -0,0 +1,67 @@ +package com.orion.ops.consts.export; + +import com.orion.ops.consts.machine.MachineFieldConst; +import com.orion.ops.entity.dto.MachineInfoImportDTO; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.function.Consumer; + +/** + * 导入类型 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/26 17:15 + */ +@Getter +@AllArgsConstructor +public enum ImportType { + + /** + * 导入机器 + */ + MACHINE(10, + "/templates/machine-import-template.xlsx", + "机器导入模板.xlsx", + MachineInfoImportDTO.class, + MachineFieldConst::validData); + + /** + * 类型 + */ + private final Integer type; + + /** + * 文件路径 + */ + private final String templatePath; + + /** + * 下载名称 + */ + private final String templateName; + + /** + * importClass + */ + private Class importClass; + + /** + * 数据验证 + */ + private Consumer valid; + + public static ImportType of(Integer type) { + if (type == null) { + return null; + } + for (ImportType value : values()) { + if (value.type.equals(type)) { + return value; + } + } + return null; + } + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/controller/DataImportController.java b/orion-ops-service/src/main/java/com/orion/ops/controller/DataImportController.java new file mode 100644 index 00000000..1caa217f --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/controller/DataImportController.java @@ -0,0 +1,142 @@ +package com.orion.ops.controller; + +import com.orion.lang.wrapper.HttpWrapper; +import com.orion.office.excel.Excels; +import com.orion.ops.OrionOpsServiceApplication; +import com.orion.ops.annotation.*; +import com.orion.ops.consts.SchedulerPools; +import com.orion.ops.consts.event.EventType; +import com.orion.ops.consts.export.ImportType; +import com.orion.ops.entity.dto.DataImportDTO; +import com.orion.ops.entity.dto.MachineInfoImportDTO; +import com.orion.ops.entity.dto.UserDTO; +import com.orion.ops.entity.request.DataImportRequest; +import com.orion.ops.entity.vo.DataImportCheckVO; +import com.orion.ops.service.api.DataImportService; +import com.orion.ops.utils.Currents; +import com.orion.ops.utils.Valid; +import com.orion.servlet.web.Servlets; +import com.orion.utils.Exceptions; +import com.orion.utils.Strings; +import com.orion.utils.Threads; +import com.orion.utils.io.Streams; +import org.apache.poi.ss.usermodel.Workbook; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.util.Date; +import java.util.List; +import java.util.function.Consumer; + +/** + * 数据导入 controller + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/26 14:02 + */ +@RestController +@RestWrapper +@RequestMapping("/orion/api/data-import") +public class DataImportController { + + @Resource + private DataImportService dataImportService; + + /** + * 获取导入模板 + */ + @GetMapping("/get-template") + @IgnoreWrapper + @IgnoreLog + @IgnoreAuth + public void getTemplate(Integer type, HttpServletResponse response) throws IOException { + ImportType importType = Valid.notNull(ImportType.of(type)); + Servlets.setDownloadHeader(response, importType.getTemplateName()); + // 读取文件 + InputStream in = OrionOpsServiceApplication.class.getResourceAsStream(importType.getTemplatePath()); + Streams.transfer(in, response.getOutputStream()); + } + + /** + * 检查导入信息 + */ + @RequestMapping("/check-data") + @SuppressWarnings("unchecked") + public DataImportCheckVO checkImportData(@RequestParam("file") MultipartFile file, + @RequestParam("type") Integer type, + @RequestParam(name = "protectPassword", required = false) String protectPassword) throws IOException { + ImportType importType = Valid.notNull(ImportType.of(type)); + Workbook workbook; + if (Strings.isBlank(protectPassword)) { + workbook = Excels.openWorkbook(file.getInputStream()); + } else { + workbook = Excels.openWorkbook(file.getInputStream(), protectPassword); + } + // 解析数据 + List rows; + try { + rows = dataImportService.parseImportWorkbook(importType, workbook); + } catch (Exception e) { + throw Exceptions.parse(e); + } finally { + Excels.close(workbook); + } + // 检查数据 + switch (importType) { + case MACHINE: + return dataImportService.checkMachineImportData((List) rows); + default: + throw Exceptions.unsupported(); + } + } + + /** + * 导入机器 + */ + @RequestMapping("/import-machine") + @EventLog(EventType.DATA_IMPORT_MACHINE) + public HttpWrapper importMachineData(@RequestBody DataImportRequest request) { + String token = Valid.notNull(request.getImportToken()); + // 导入 + this.asyncImportData(token, d -> dataImportService.importMachineData(d)); + return HttpWrapper.ok(); + } + + /** + * 取消导入 + */ + @RequestMapping("/cancel-import") + public HttpWrapper cancelImportData(@RequestBody DataImportRequest request) { + String token = request.getImportToken(); + if (Strings.isBlank(token)) { + return HttpWrapper.ok(); + } + dataImportService.clearImportToken(token); + return HttpWrapper.ok(); + } + + /** + * 异步导入 + * + * @param token token + * @param handler 处理器 + */ + private void asyncImportData(String token, Consumer handler) { + // 获取 token 信息 + DataImportDTO importData = dataImportService.checkImportToken(token); + UserDTO user = Currents.getUser(); + importData.setUserId(user.getId()); + importData.setUserName(user.getUsername()); + importData.setImportTime(new Date()); + // 异步执行导入 + Threads.start(() -> handler.accept(importData), SchedulerPools.ASYNC_IMPORT_SCHEDULER); + // 清空token + dataImportService.clearImportToken(token); + } + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/entity/dto/BaseDataImportDTO.java b/orion-ops-service/src/main/java/com/orion/ops/entity/dto/BaseDataImportDTO.java new file mode 100644 index 00000000..58c0d00e --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/entity/dto/BaseDataImportDTO.java @@ -0,0 +1,20 @@ +package com.orion.ops.entity.dto; + +import lombok.Data; + +/** + * 导入数据 基类 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/26 15:27 + */ +@Data +public class BaseDataImportDTO { + + /** + * 非法信息 + */ + private String illegalMessage; + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/entity/dto/DataImportDTO.java b/orion-ops-service/src/main/java/com/orion/ops/entity/dto/DataImportDTO.java new file mode 100644 index 00000000..3613bfba --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/entity/dto/DataImportDTO.java @@ -0,0 +1,55 @@ +package com.orion.ops.entity.dto; + +import com.orion.ops.entity.vo.DataImportCheckVO; +import lombok.Data; + +import java.util.Date; + +/** + * 数据导入对象 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/26 17:42 + */ +@Data +public class DataImportDTO { + + /** + * token + */ + private String importToken; + + /** + * 类型 + * + * @see com.orion.ops.consts.export.ImportType + */ + private Integer type; + + /** + * 导入数据 json + */ + private String data; + + /** + * 检查数据 + */ + private DataImportCheckVO check; + + /** + * 导入用户id 手动 + */ + private Long userId; + + /** + * 导入用户名称 手动 + */ + private String userName; + + /** + * 导入时间 手动 + */ + private Date importTime; + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/entity/request/DataImportRequest.java b/orion-ops-service/src/main/java/com/orion/ops/entity/request/DataImportRequest.java new file mode 100644 index 00000000..1d3be88a --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/entity/request/DataImportRequest.java @@ -0,0 +1,20 @@ +package com.orion.ops.entity.request; + +import lombok.Data; + +/** + * 数据导入请求 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/26 17:33 + */ +@Data +public class DataImportRequest { + + /** + * 导入 token + */ + private String importToken; + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/entity/vo/DataImportCheckRowVO.java b/orion-ops-service/src/main/java/com/orion/ops/entity/vo/DataImportCheckRowVO.java new file mode 100644 index 00000000..d552e54b --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/entity/vo/DataImportCheckRowVO.java @@ -0,0 +1,35 @@ +package com.orion.ops.entity.vo; + +import lombok.Data; + +/** + * 数据导入检查 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/27 11:18 + */ +@Data +public class DataImportCheckRowVO { + + /** + * 索引 0开始 + */ + private Integer index; + + /** + * 行号 前端提示 + */ + private Integer row; + + /** + * 唯一标识 + */ + private String symbol; + + /** + * 非法信息 + */ + private String illegalMessage; + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/entity/vo/DataImportCheckVO.java b/orion-ops-service/src/main/java/com/orion/ops/entity/vo/DataImportCheckVO.java new file mode 100644 index 00000000..4c2ca78f --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/entity/vo/DataImportCheckVO.java @@ -0,0 +1,37 @@ +package com.orion.ops.entity.vo; + +import lombok.Data; + +import java.util.List; + +/** + * 数据导入检查 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/27 11:18 + */ +@Data +public class DataImportCheckVO { + + /** + * 无效行 + */ + private List illegalRows; + + /** + * 插入行 + */ + private List insertRows; + + /** + * 更新行 + */ + private List updateRows; + + /** + * 导入 token + */ + private String importToken; + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/service/api/DataImportService.java b/orion-ops-service/src/main/java/com/orion/ops/service/api/DataImportService.java new file mode 100644 index 00000000..2bf61280 --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/service/api/DataImportService.java @@ -0,0 +1,60 @@ +package com.orion.ops.service.api; + +import com.orion.ops.consts.export.ImportType; +import com.orion.ops.entity.dto.DataImportDTO; +import com.orion.ops.entity.dto.MachineInfoImportDTO; +import com.orion.ops.entity.vo.DataImportCheckVO; +import org.apache.poi.ss.usermodel.Workbook; + +import java.util.List; + +/** + * 数据导入服务 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/26 17:06 + */ +public interface DataImportService { + + /** + * 解析导入 workbook + * + * @param workbook workbook + * @param type type + * @param T + * @return list + */ + List parseImportWorkbook(ImportType type, Workbook workbook); + + /** + * 检查机器导入信息 + * + * @param rows rows + * @return 导入信息 + */ + DataImportCheckVO checkMachineImportData(List rows); + + /** + * 检查导入 token + * + * @param token token + * @return 导入数据 + */ + DataImportDTO checkImportToken(String token); + + /** + * 导入机器信息 + * + * @param checkData 缓存信息 + */ + void importMachineData(DataImportDTO checkData); + + /** + * 清空导入 token + * + * @param token token + */ + void clearImportToken(String token); + +} diff --git a/orion-ops-service/src/main/java/com/orion/ops/service/impl/DataImportServiceImpl.java b/orion-ops-service/src/main/java/com/orion/ops/service/impl/DataImportServiceImpl.java new file mode 100644 index 00000000..b6b92b4a --- /dev/null +++ b/orion-ops-service/src/main/java/com/orion/ops/service/impl/DataImportServiceImpl.java @@ -0,0 +1,256 @@ +package com.orion.ops.service.impl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.orion.id.UUIds; +import com.orion.office.excel.reader.ExcelBeanReader; +import com.orion.ops.consts.KeyConst; +import com.orion.ops.consts.MessageConst; +import com.orion.ops.consts.event.EventKeys; +import com.orion.ops.consts.export.ImportType; +import com.orion.ops.consts.message.MessageType; +import com.orion.ops.consts.system.SystemEnvAttr; +import com.orion.ops.dao.MachineInfoDAO; +import com.orion.ops.entity.domain.MachineInfoDO; +import com.orion.ops.entity.dto.BaseDataImportDTO; +import com.orion.ops.entity.dto.DataImportDTO; +import com.orion.ops.entity.dto.MachineInfoImportDTO; +import com.orion.ops.entity.vo.DataImportCheckRowVO; +import com.orion.ops.entity.vo.DataImportCheckVO; +import com.orion.ops.service.api.DataImportService; +import com.orion.ops.service.api.WebSideMessageService; +import com.orion.ops.utils.Currents; +import com.orion.ops.utils.PathBuilders; +import com.orion.utils.Exceptions; +import com.orion.utils.Strings; +import com.orion.utils.Valid; +import com.orion.utils.collect.Lists; +import com.orion.utils.collect.Maps; +import com.orion.utils.convert.Converts; +import com.orion.utils.io.FileWriters; +import com.orion.utils.io.Files1; +import com.orion.utils.time.Dates; +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.ss.usermodel.Workbook; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * 数据导入服务 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2022/5/26 17:07 + */ +@Slf4j +@Service("dataImportService") +public class DataImportServiceImpl implements DataImportService { + + @Resource + private WebSideMessageService webSideMessageService; + + @Resource + private MachineInfoDAO machineInfoDAO; + + @Resource + private RedisTemplate redisTemplate; + + @Override + @SuppressWarnings("unchecked") + public List parseImportWorkbook(ImportType type, Workbook workbook) { + ExcelBeanReader reader = new ExcelBeanReader(workbook, workbook.getSheetAt(0), (Class) type.getImportClass()); + return reader.skip(2) + .read() + .getRows(); + } + + @Override + public DataImportCheckVO checkMachineImportData(List rows) { + // 检查数据合法性 + this.validImportData(ImportType.MACHINE, rows); + // 设置检查对象 + String dataJson = JSON.toJSONString(rows); + List illegalRows = Lists.newList(); + List insertRows = Lists.newList(); + List updateRows = Lists.newList(); + // 通过 tag 查询机器 + List tagList = rows.stream() + .filter(s -> Objects.isNull(s.getIllegalMessage())) + .map(MachineInfoImportDTO::getTag) + .collect(Collectors.toList()); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + .in(MachineInfoDO::getMachineTag, tagList); + List machines = machineInfoDAO.selectList(wrapper); + // 设置数据 + for (int i = 0; i < rows.size(); i++) { + MachineInfoImportDTO row = rows.get(i); + // 设置检查数据 + DataImportCheckRowVO checkRow = Converts.to(row, DataImportCheckRowVO.class); + checkRow.setIndex(i); + checkRow.setRow(i + 3); + // 不合法数据 + if (checkRow.getIllegalMessage() != null) { + illegalRows.add(checkRow); + continue; + } + final boolean update = machines.stream().anyMatch(m -> m.getMachineTag().equals(row.getTag())); + if (update) { + updateRows.add(checkRow); + } else { + insertRows.add(checkRow); + } + } + // 返回数据 + return this.setCheckData(ImportType.MACHINE.getType(), dataJson, illegalRows, insertRows, updateRows); + } + + @Override + public DataImportDTO checkImportToken(String token) { + // 查询缓存 + String data = redisTemplate.opsForValue().get(Strings.format(KeyConst.DATA_IMPORT_TOKEN, Currents.getUserId(), token)); + if (Strings.isEmpty(data)) { + throw Exceptions.argument(MessageConst.OPERATOR_TIMEOUT); + } + return JSON.parseObject(data, DataImportDTO.class); + } + + @Override + public void importMachineData(DataImportDTO importData) { + Exception ex = null; + try { + // 获取缓存数据 + List rows = this.getImportData(importData); + DataImportCheckVO dataCheck = importData.getCheck(); + // 插入 + List insertRows = dataCheck.getInsertRows(); + for (DataImportCheckRowVO insertRow : insertRows) { + // 获取需要插入的数据 + MachineInfoImportDTO insertImportRow = rows.get(insertRow.getIndex()); + MachineInfoDO insert = Converts.to(insertImportRow, MachineInfoDO.class); + machineInfoDAO.insert(insert); + } + // 更新 + List updateRows = dataCheck.getUpdateRows(); + for (DataImportCheckRowVO updateRow : updateRows) { + // 获取需要更新的数据 + MachineInfoImportDTO updateImportRow = rows.get(updateRow.getIndex()); + MachineInfoDO update = Converts.to(updateImportRow, MachineInfoDO.class); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + .eq(MachineInfoDO::getMachineTag, update.getMachineTag()); + machineInfoDAO.update(update, wrapper); + } + } catch (Exception e) { + ex = e; + log.error("机器信息导入失败 token: {}, data: {}", importData.getImportToken(), JSON.toJSONString(importData), e); + } + // 发送站内信 + this.sendImportWebSideMessage(importData, ex == null ? MessageType.MACHINE_IMPORT_SUCCESS : MessageType.MACHINE_IMPORT_FAILURE); + // 保存日志 + this.saveImportDataJson(importData); + } + + @Override + public void clearImportToken(String token) { + redisTemplate.delete(Strings.format(KeyConst.DATA_IMPORT_TOKEN, Currents.getUserId(), token)); + } + + /** + * 验证对象合法性 + * + * @param importType importType + * @param rows rows + */ + private void validImportData(ImportType importType, List rows) { + for (BaseDataImportDTO row : rows) { + try { + importType.getValid().accept(row); + } catch (Exception e) { + row.setIllegalMessage(e.getMessage()); + } + } + + } + + /** + * 设置检查数据 + * + * @param type type + * @param dataJson json + * @param illegalRows 无效数据 + * @param insertRows 插入数据 + * @param updateRows 更新数据 + * @return 检查数据 + */ + private DataImportCheckVO setCheckData(Integer type, + String dataJson, + List illegalRows, + List insertRows, + List updateRows) { + String token = UUIds.random32(); + String cacheKey = Strings.format(KeyConst.DATA_IMPORT_TOKEN, Currents.getUserId(), token); + // 返回数据 + DataImportCheckVO check = new DataImportCheckVO(); + check.setImportToken(token); + check.setIllegalRows(illegalRows); + check.setInsertRows(insertRows); + check.setUpdateRows(updateRows); + // 设置缓存 + DataImportDTO cache = new DataImportDTO(); + cache.setImportToken(token); + cache.setType(type); + cache.setData(dataJson); + cache.setCheck(check); + redisTemplate.opsForValue().set(cacheKey, JSON.toJSONString(cache), + KeyConst.DATA_IMPORT_TOKEN_EXPIRE, TimeUnit.SECONDS); + return check; + } + + /** + * 获取导入数据 + * + * @param data data + * @param T + * @return list + */ + @SuppressWarnings("unchecked") + private List getImportData(DataImportDTO data) { + ImportType type = ImportType.of(data.getType()); + Valid.notNull(type); + return (List) JSON.parseArray(data.getData(), type.getImportClass()); + } + + /** + * 发送站内信 + * + * @param importData importData + * @param type type + */ + private void sendImportWebSideMessage(DataImportDTO importData, MessageType type) { + // 站内信参数 + Map params = Maps.newMap(); + params.put(EventKeys.TIME, Dates.format(importData.getImportTime())); + params.put(EventKeys.TOKEN, importData.getImportToken()); + webSideMessageService.addMessage(type, importData.getUserId(), importData.getUserName(), params); + } + + /** + * 将导入 json 存储到本地 + * + * @param importData json + */ + private void saveImportDataJson(DataImportDTO importData) { + // 将数据存储到本地 log + String importJsonPath = PathBuilders.getImportDataJsonPath(importData.getUserId(), importData.getType(), importData.getImportToken()); + String path = Files1.getPath(SystemEnvAttr.LOG_PATH.getValue(), importJsonPath); + FileWriters.write(path, JSON.toJSONString(importData, SerializerFeature.PrettyFormat)); + } + +} diff --git a/orion-ops-service/src/main/resources/templates/machine-import-template.xlsx b/orion-ops-service/src/main/resources/templates/machine-import-template.xlsx new file mode 100644 index 00000000..092eaf6e Binary files /dev/null and b/orion-ops-service/src/main/resources/templates/machine-import-template.xlsx differ