opensource/orion-ops
3
0
Fork 0
mirror of https://gitee.com/lijiahangmax/orion-ops.git synced 2026-09-24 15:30:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: import machine data.

Browse Source
This commit is contained in:
lijiahang
2022-05-29 13:45:50 +08:00
parent c60c8d1539
commit d86df6ef54
10 changed files with 692 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
orion-ops-service/src/main/java/com/orion/ops/consts/export/ImportType.java
+67
View File
@@ -0,0 +1,67 @@
package com.orion.ops.consts.export;
import com.orion.ops.consts.machine.MachineFieldConst;
import com.orion.ops.entity.dto.MachineInfoImportDTO;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.function.Consumer;
/**
* 导入类型
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/26 17:15
*/
@Getter
@AllArgsConstructor
public enum ImportType {
/**
* 导入机器
*/
MACHINE(10,
"/templates/machine-import-template.xlsx",
"机器导入模板.xlsx",
MachineInfoImportDTO.class,
MachineFieldConst::validData);
/**
* 类型
*/
private final Integer type;
/**
* 文件路径
*/
private final String templatePath;
/**
* 下载名称
*/
private final String templateName;
/**
* importClass
*/
private Class<?> importClass;
/**
* 数据验证
*/
private Consumer<Object> valid;
public static ImportType of(Integer type) {
if (type == null) {
return null;
}
for (ImportType value : values()) {
if (value.type.equals(type)) {
return value;
}
}
return null;
}
}
orion-ops-service/src/main/java/com/orion/ops/controller/DataImportController.java
+142
View File
@@ -0,0 +1,142 @@
package com.orion.ops.controller;
import com.orion.lang.wrapper.HttpWrapper;
import com.orion.office.excel.Excels;
import com.orion.ops.OrionOpsServiceApplication;
import com.orion.ops.annotation.*;
import com.orion.ops.consts.SchedulerPools;
import com.orion.ops.consts.event.EventType;
import com.orion.ops.consts.export.ImportType;
import com.orion.ops.entity.dto.DataImportDTO;
import com.orion.ops.entity.dto.MachineInfoImportDTO;
import com.orion.ops.entity.dto.UserDTO;
import com.orion.ops.entity.request.DataImportRequest;
import com.orion.ops.entity.vo.DataImportCheckVO;
import com.orion.ops.service.api.DataImportService;
import com.orion.ops.utils.Currents;
import com.orion.ops.utils.Valid;
import com.orion.servlet.web.Servlets;
import com.orion.utils.Exceptions;
import com.orion.utils.Strings;
import com.orion.utils.Threads;
import com.orion.utils.io.Streams;
import org.apache.poi.ss.usermodel.Workbook;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.List;
import java.util.function.Consumer;
/**
* 数据导入 controller
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/26 14:02
*/
@RestController
@RestWrapper
@RequestMapping("/orion/api/data-import")
public class DataImportController {
@Resource
private DataImportService dataImportService;
/**
* 获取导入模板
*/
@GetMapping("/get-template")
@IgnoreWrapper
@IgnoreLog
@IgnoreAuth
public void getTemplate(Integer type, HttpServletResponse response) throws IOException {
ImportType importType = Valid.notNull(ImportType.of(type));
Servlets.setDownloadHeader(response, importType.getTemplateName());
// 读取文件
InputStream in = OrionOpsServiceApplication.class.getResourceAsStream(importType.getTemplatePath());
Streams.transfer(in, response.getOutputStream());
}
/**
* 检查导入信息
*/
@RequestMapping("/check-data")
@SuppressWarnings("unchecked")
public DataImportCheckVO checkImportData(@RequestParam("file") MultipartFile file,
@RequestParam("type") Integer type,
@RequestParam(name = "protectPassword", required = false) String protectPassword) throws IOException {
ImportType importType = Valid.notNull(ImportType.of(type));
Workbook workbook;
if (Strings.isBlank(protectPassword)) {
workbook = Excels.openWorkbook(file.getInputStream());
} else {
workbook = Excels.openWorkbook(file.getInputStream(), protectPassword);
}
// 解析数据
List<?> rows;
try {
rows = dataImportService.parseImportWorkbook(importType, workbook);
} catch (Exception e) {
throw Exceptions.parse(e);
} finally {
Excels.close(workbook);
}
// 检查数据
switch (importType) {
case MACHINE:
return dataImportService.checkMachineImportData((List<MachineInfoImportDTO>) rows);
default:
throw Exceptions.unsupported();
}
}
/**
* 导入机器
*/
@RequestMapping("/import-machine")
@EventLog(EventType.DATA_IMPORT_MACHINE)
public HttpWrapper<?> importMachineData(@RequestBody DataImportRequest request) {
String token = Valid.notNull(request.getImportToken());
// 导入
this.asyncImportData(token, d -> dataImportService.importMachineData(d));
return HttpWrapper.ok();
}
/**
* 取消导入
*/
@RequestMapping("/cancel-import")
public HttpWrapper<?> cancelImportData(@RequestBody DataImportRequest request) {
String token = request.getImportToken();
if (Strings.isBlank(token)) {
return HttpWrapper.ok();
}
dataImportService.clearImportToken(token);
return HttpWrapper.ok();
}
/**
* 异步导入
*
* @param token token
* @param handler 处理器
*/
private void asyncImportData(String token, Consumer<DataImportDTO> handler) {
// 获取 token 信息
DataImportDTO importData = dataImportService.checkImportToken(token);
UserDTO user = Currents.getUser();
importData.setUserId(user.getId());
importData.setUserName(user.getUsername());
importData.setImportTime(new Date());
// 异步执行导入
Threads.start(() -> handler.accept(importData), SchedulerPools.ASYNC_IMPORT_SCHEDULER);
// 清空token
dataImportService.clearImportToken(token);
}
}
orion-ops-service/src/main/java/com/orion/ops/entity/dto/BaseDataImportDTO.java
+20
View File
@@ -0,0 +1,20 @@
package com.orion.ops.entity.dto;
import lombok.Data;
/**
* 导入数据 基类
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/26 15:27
*/
@Data
public class BaseDataImportDTO {
/**
* 非法信息
*/
private String illegalMessage;
}
orion-ops-service/src/main/java/com/orion/ops/entity/dto/DataImportDTO.java
+55
View File
@@ -0,0 +1,55 @@
package com.orion.ops.entity.dto;
import com.orion.ops.entity.vo.DataImportCheckVO;
import lombok.Data;
import java.util.Date;
/**
* 数据导入对象
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/26 17:42
*/
@Data
public class DataImportDTO {
/**
* token
*/
private String importToken;
/**
* 类型
*
* @see com.orion.ops.consts.export.ImportType
*/
private Integer type;
/**
* 导入数据 json
*/
private String data;
/**
* 检查数据
*/
private DataImportCheckVO check;
/**
* 导入用户id 手动
*/
private Long userId;
/**
* 导入用户名称 手动
*/
private String userName;
/**
* 导入时间 手动
*/
private Date importTime;
}
orion-ops-service/src/main/java/com/orion/ops/entity/request/DataImportRequest.java
+20
View File
@@ -0,0 +1,20 @@
package com.orion.ops.entity.request;
import lombok.Data;
/**
* 数据导入请求
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/26 17:33
*/
@Data
public class DataImportRequest {
/**
* 导入 token
*/
private String importToken;
}
orion-ops-service/src/main/java/com/orion/ops/entity/vo/DataImportCheckRowVO.java
+35
View File
@@ -0,0 +1,35 @@
package com.orion.ops.entity.vo;
import lombok.Data;
/**
* 数据导入检查
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/27 11:18
*/
@Data
public class DataImportCheckRowVO {
/**
* 索引 0开始
*/
private Integer index;
/**
* 行号 前端提示
*/
private Integer row;
/**
* 唯一标识
*/
private String symbol;
/**
* 非法信息
*/
private String illegalMessage;
}
orion-ops-service/src/main/java/com/orion/ops/entity/vo/DataImportCheckVO.java
+37
View File
@@ -0,0 +1,37 @@
package com.orion.ops.entity.vo;
import lombok.Data;
import java.util.List;
/**
* 数据导入检查
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/27 11:18
*/
@Data
public class DataImportCheckVO {
/**
* 无效行
*/
private List<DataImportCheckRowVO> illegalRows;
/**
* 插入行
*/
private List<DataImportCheckRowVO> insertRows;
/**
* 更新行
*/
private List<DataImportCheckRowVO> updateRows;
/**
* 导入 token
*/
private String importToken;
}
orion-ops-service/src/main/java/com/orion/ops/service/api/DataImportService.java
+60
View File
@@ -0,0 +1,60 @@
package com.orion.ops.service.api;
import com.orion.ops.consts.export.ImportType;
import com.orion.ops.entity.dto.DataImportDTO;
import com.orion.ops.entity.dto.MachineInfoImportDTO;
import com.orion.ops.entity.vo.DataImportCheckVO;
import org.apache.poi.ss.usermodel.Workbook;
import java.util.List;
/**
* 数据导入服务
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/26 17:06
*/
public interface DataImportService {
/**
* 解析导入 workbook
*
* @param workbook workbook
* @param type type
* @param <T> T
* @return list
*/
<T> List<T> parseImportWorkbook(ImportType type, Workbook workbook);
/**
* 检查机器导入信息
*
* @param rows rows
* @return 导入信息
*/
DataImportCheckVO checkMachineImportData(List<MachineInfoImportDTO> rows);
/**
* 检查导入 token
*
* @param token token
* @return 导入数据
*/
DataImportDTO checkImportToken(String token);
/**
* 导入机器信息
*
* @param checkData 缓存信息
*/
void importMachineData(DataImportDTO checkData);
/**
* 清空导入 token
*
* @param token token
*/
void clearImportToken(String token);
}
orion-ops-service/src/main/java/com/orion/ops/service/impl/DataImportServiceImpl.java
+256
View File
@@ -0,0 +1,256 @@
package com.orion.ops.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.orion.id.UUIds;
import com.orion.office.excel.reader.ExcelBeanReader;
import com.orion.ops.consts.KeyConst;
import com.orion.ops.consts.MessageConst;
import com.orion.ops.consts.event.EventKeys;
import com.orion.ops.consts.export.ImportType;
import com.orion.ops.consts.message.MessageType;
import com.orion.ops.consts.system.SystemEnvAttr;
import com.orion.ops.dao.MachineInfoDAO;
import com.orion.ops.entity.domain.MachineInfoDO;
import com.orion.ops.entity.dto.BaseDataImportDTO;
import com.orion.ops.entity.dto.DataImportDTO;
import com.orion.ops.entity.dto.MachineInfoImportDTO;
import com.orion.ops.entity.vo.DataImportCheckRowVO;
import com.orion.ops.entity.vo.DataImportCheckVO;
import com.orion.ops.service.api.DataImportService;
import com.orion.ops.service.api.WebSideMessageService;
import com.orion.ops.utils.Currents;
import com.orion.ops.utils.PathBuilders;
import com.orion.utils.Exceptions;
import com.orion.utils.Strings;
import com.orion.utils.Valid;
import com.orion.utils.collect.Lists;
import com.orion.utils.collect.Maps;
import com.orion.utils.convert.Converts;
import com.orion.utils.io.FileWriters;
import com.orion.utils.io.Files1;
import com.orion.utils.time.Dates;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Workbook;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* 数据导入服务
*
* @author Jiahang Li
* @version 1.0.0
* @since 2022/5/26 17:07
*/
@Slf4j
@Service("dataImportService")
public class DataImportServiceImpl implements DataImportService {
@Resource
private WebSideMessageService webSideMessageService;
@Resource
private MachineInfoDAO machineInfoDAO;
@Resource
private RedisTemplate<String, String> redisTemplate;
@Override
@SuppressWarnings("unchecked")
public <T> List<T> parseImportWorkbook(ImportType type, Workbook workbook) {
ExcelBeanReader<T> reader = new ExcelBeanReader<T>(workbook, workbook.getSheetAt(0), (Class<T>) type.getImportClass());
return reader.skip(2)
.read()
.getRows();
}
@Override
public DataImportCheckVO checkMachineImportData(List<MachineInfoImportDTO> rows) {
// 检查数据合法性
this.validImportData(ImportType.MACHINE, rows);
// 设置检查对象
String dataJson = JSON.toJSONString(rows);
List<DataImportCheckRowVO> illegalRows = Lists.newList();
List<DataImportCheckRowVO> insertRows = Lists.newList();
List<DataImportCheckRowVO> updateRows = Lists.newList();
// 通过 tag 查询机器
List<String> tagList = rows.stream()
.filter(s -> Objects.isNull(s.getIllegalMessage()))
.map(MachineInfoImportDTO::getTag)
.collect(Collectors.toList());
LambdaQueryWrapper<MachineInfoDO> wrapper = new LambdaQueryWrapper<MachineInfoDO>()
.in(MachineInfoDO::getMachineTag, tagList);
List<MachineInfoDO> machines = machineInfoDAO.selectList(wrapper);
// 设置数据
for (int i = 0; i < rows.size(); i++) {
MachineInfoImportDTO row = rows.get(i);
// 设置检查数据
DataImportCheckRowVO checkRow = Converts.to(row, DataImportCheckRowVO.class);
checkRow.setIndex(i);
checkRow.setRow(i + 3);
// 不合法数据
if (checkRow.getIllegalMessage() != null) {
illegalRows.add(checkRow);
continue;
}
final boolean update = machines.stream().anyMatch(m -> m.getMachineTag().equals(row.getTag()));
if (update) {
updateRows.add(checkRow);
} else {
insertRows.add(checkRow);
}
}
// 返回数据
return this.setCheckData(ImportType.MACHINE.getType(), dataJson, illegalRows, insertRows, updateRows);
}
@Override
public DataImportDTO checkImportToken(String token) {
// 查询缓存
String data = redisTemplate.opsForValue().get(Strings.format(KeyConst.DATA_IMPORT_TOKEN, Currents.getUserId(), token));
if (Strings.isEmpty(data)) {
throw Exceptions.argument(MessageConst.OPERATOR_TIMEOUT);
}
return JSON.parseObject(data, DataImportDTO.class);
}
@Override
public void importMachineData(DataImportDTO importData) {
Exception ex = null;
try {
// 获取缓存数据
List<MachineInfoImportDTO> rows = this.getImportData(importData);
DataImportCheckVO dataCheck = importData.getCheck();
// 插入
List<DataImportCheckRowVO> insertRows = dataCheck.getInsertRows();
for (DataImportCheckRowVO insertRow : insertRows) {
// 获取需要插入的数据
MachineInfoImportDTO insertImportRow = rows.get(insertRow.getIndex());
MachineInfoDO insert = Converts.to(insertImportRow, MachineInfoDO.class);
machineInfoDAO.insert(insert);
}
// 更新
List<DataImportCheckRowVO> updateRows = dataCheck.getUpdateRows();
for (DataImportCheckRowVO updateRow : updateRows) {
// 获取需要更新的数据
MachineInfoImportDTO updateImportRow = rows.get(updateRow.getIndex());
MachineInfoDO update = Converts.to(updateImportRow, MachineInfoDO.class);
LambdaQueryWrapper<MachineInfoDO> wrapper = new LambdaQueryWrapper<MachineInfoDO>()
.eq(MachineInfoDO::getMachineTag, update.getMachineTag());
machineInfoDAO.update(update, wrapper);
}
} catch (Exception e) {
ex = e;
log.error("机器信息导入失败 token: {}, data: {}", importData.getImportToken(), JSON.toJSONString(importData), e);
}
// 发送站内信
this.sendImportWebSideMessage(importData, ex == null ? MessageType.MACHINE_IMPORT_SUCCESS : MessageType.MACHINE_IMPORT_FAILURE);
// 保存日志
this.saveImportDataJson(importData);
}
@Override
public void clearImportToken(String token) {
redisTemplate.delete(Strings.format(KeyConst.DATA_IMPORT_TOKEN, Currents.getUserId(), token));
}
/**
* 验证对象合法性
*
* @param importType importType
* @param rows rows
*/
private void validImportData(ImportType importType, List<? extends BaseDataImportDTO> rows) {
for (BaseDataImportDTO row : rows) {
try {
importType.getValid().accept(row);
} catch (Exception e) {
row.setIllegalMessage(e.getMessage());
}
}
}
/**
* 设置检查数据
*
* @param type type
* @param dataJson json
* @param illegalRows 无效数据
* @param insertRows 插入数据
* @param updateRows 更新数据
* @return 检查数据
*/
private DataImportCheckVO setCheckData(Integer type,
String dataJson,
List<DataImportCheckRowVO> illegalRows,
List<DataImportCheckRowVO> insertRows,
List<DataImportCheckRowVO> updateRows) {
String token = UUIds.random32();
String cacheKey = Strings.format(KeyConst.DATA_IMPORT_TOKEN, Currents.getUserId(), token);
// 返回数据
DataImportCheckVO check = new DataImportCheckVO();
check.setImportToken(token);
check.setIllegalRows(illegalRows);
check.setInsertRows(insertRows);
check.setUpdateRows(updateRows);
// 设置缓存
DataImportDTO cache = new DataImportDTO();
cache.setImportToken(token);
cache.setType(type);
cache.setData(dataJson);
cache.setCheck(check);
redisTemplate.opsForValue().set(cacheKey, JSON.toJSONString(cache),
KeyConst.DATA_IMPORT_TOKEN_EXPIRE, TimeUnit.SECONDS);
return check;
}
/**
* 获取导入数据
*
* @param data data
* @param <T> T
* @return list
*/
@SuppressWarnings("unchecked")
private <T extends BaseDataImportDTO> List<T> getImportData(DataImportDTO data) {
ImportType type = ImportType.of(data.getType());
Valid.notNull(type);
return (List<T>) JSON.parseArray(data.getData(), type.getImportClass());
}
/**
* 发送站内信
*
* @param importData importData
* @param type type
*/
private void sendImportWebSideMessage(DataImportDTO importData, MessageType type) {
// 站内信参数
Map<String, Object> params = Maps.newMap();
params.put(EventKeys.TIME, Dates.format(importData.getImportTime()));
params.put(EventKeys.TOKEN, importData.getImportToken());
webSideMessageService.addMessage(type, importData.getUserId(), importData.getUserName(), params);
}
/**
* 将导入 json 存储到本地
*
* @param importData json
*/
private void saveImportDataJson(DataImportDTO importData) {
// 将数据存储到本地 log
String importJsonPath = PathBuilders.getImportDataJsonPath(importData.getUserId(), importData.getType(), importData.getImportToken());
String path = Files1.getPath(SystemEnvAttr.LOG_PATH.getValue(), importJsonPath);
FileWriters.write(path, JSON.toJSONString(importData, SerializerFeature.PrettyFormat));
}
}
orion-ops-service/src/main/resources/templates/machine-import-template.xlsx
BIN
View File
Binary file not shown.