gtsoft/gtsoft-snail-job-server
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects 1 Releases Wiki Activity

feat(sj_1.0.0): 优化重试场景的导出

Browse Source
This commit is contained in:
opensnail 2024-05-29 23:10:10 +08:00
parent fe4a69c66f
commit b80da2d616
9 changed files with 132 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/controller/GroupConfigController.java
View File

@ -93,10 +93,6 @@ public class GroupConfigController {
@PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@LoginRequired(role = RoleEnum.ADMIN)
public void importScene(@RequestPart("file") MultipartFile file) throws IOException {
if (file.isEmpty()) {
throw new SnailJobCommonException("请选择一个文件上传");
}
groupConfigService.importGroup(ImportUtils.parseList(file, GroupConfigRequestVO.class));
}

13
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/controller/JobController.java
View File

@ -101,21 +101,16 @@ public class JobController {
}
@PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@LoginRequired(role = RoleEnum.ADMIN)
@LoginRequired
public void importScene(@RequestPart("file") MultipartFile file) throws IOException {
if (file.isEmpty()) {
throw new SnailJobCommonException("请选择一个文件上传");
}
List<JobRequestVO> requestList = ImportUtils.parseList(file, JobRequestVO.class);
jobService.importJobs(requestList);
jobService.importJobs(ImportUtils.parseList(file, JobRequestVO.class));
}
@PostMapping("/export")
@LoginRequired(role = RoleEnum.ADMIN)
@LoginRequired
@OriginalControllerReturnValue
public ResponseEntity<String> exportGroup(@RequestBody Set<Long> jobIds) {
String jobsJson = jobService.exportJobs(jobIds);
return ExportUtils.doExport(jobsJson);
return ExportUtils.doExport(jobService.exportJobs(jobIds));
}
}

10
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/controller/SceneConfigController.java
View File

@ -4,6 +4,7 @@ import com.aizuda.snailjob.common.core.annotation.OriginalControllerReturnValue;
import com.aizuda.snailjob.common.core.exception.SnailJobCommonException;
import com.aizuda.snailjob.server.web.annotation.LoginRequired;
import com.aizuda.snailjob.server.web.model.base.PageResult;
import com.aizuda.snailjob.server.web.model.request.ExportSceneVO;
import com.aizuda.snailjob.server.web.model.request.SceneConfigQueryVO;
import com.aizuda.snailjob.server.web.model.request.SceneConfigRequestVO;
import com.aizuda.snailjob.server.web.model.response.SceneConfigResponseVO;
@ -72,10 +73,6 @@ public class SceneConfigController {
@LoginRequired
@PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public void importScene(@RequestPart("file") MultipartFile file) throws IOException {
if (file.isEmpty()) {
throw new SnailJobCommonException("请选择一个文件上传");
}
// 写入数据
sceneConfigService.importSceneConfig(ImportUtils.parseList(file, SceneConfigRequestVO.class));
}
@ -83,9 +80,8 @@ public class SceneConfigController {
@LoginRequired
@PostMapping("/export")
@OriginalControllerReturnValue
public ResponseEntity<String> export(@RequestBody Set<Long> sceneIds) {
return ExportUtils.doExport(sceneConfigService.exportSceneConfig(sceneIds));
public ResponseEntity<String> export(@RequestBody ExportSceneVO exportSceneVO) {
return ExportUtils.doExport(sceneConfigService.exportSceneConfig(exportSceneVO));
}
}

23
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/request/ExportSceneVO.java Normal file
View File

@ -0,0 +1,23 @@
package com.aizuda.snailjob.server.web.model.request;
import lombok.Data;
import java.util.Set;
/**
* @author: opensnail
* @date : 2024-05-29
* @since : sj_1.0.0
*/
@Data
public class ExportSceneVO {
private String groupName;
private Integer sceneStatus;
private String sceneName;
private Set<Long> sceneIds;
}

1
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/request/SceneConfigQueryVO.java
View File

@ -14,4 +14,5 @@ import lombok.EqualsAndHashCode;
public class SceneConfigQueryVO extends BaseQueryVO {
private String groupName;
private String sceneName;
private Integer sceneStatus;
}

4
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/SceneConfigService.java
View File

@ -1,6 +1,7 @@
package com.aizuda.snailjob.server.web.service;
import com.aizuda.snailjob.server.web.model.base.PageResult;
import com.aizuda.snailjob.server.web.model.request.ExportSceneVO;
import com.aizuda.snailjob.server.web.model.request.SceneConfigQueryVO;
import com.aizuda.snailjob.server.web.model.request.SceneConfigRequestVO;
import com.aizuda.snailjob.server.web.model.response.SceneConfigResponseVO;
@ -8,7 +9,6 @@ import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import java.util.List;
import java.util.Set;
/**
* @author: opensnail
@ -30,6 +30,6 @@ public interface SceneConfigService {
void importSceneConfig(@Valid @NotEmpty(message = "导入数据不能为空") List<SceneConfigRequestVO> requests);
String exportSceneConfig(Set<Long> sceneIds);
String exportSceneConfig(ExportSceneVO exportSceneVO);
}

3
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/impl/GroupConfigServiceImpl.java
View File

@ -439,8 +439,7 @@ public class GroupConfigServiceImpl implements GroupConfigService {
return groupConfigs.stream().map(GroupConfigPartitionTask::new).toList();
}), partitionTasks -> {
List<GroupConfigPartitionTask> configPartitionTasks = (List<GroupConfigPartitionTask>) partitionTasks;
List<GroupConfig> configs = configPartitionTasks.stream().map(GroupConfigPartitionTask::getConfig)
.collect(Collectors.toList());
List<GroupConfig> configs = StreamUtils.toList(configPartitionTasks, GroupConfigPartitionTask::getConfig);
allRequestList.addAll(GroupConfigConverter.INSTANCE.toGroupConfigRequestVOs(configs));
}, 0);

55
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/impl/SceneConfigServiceImpl.java
View File

@ -5,10 +5,13 @@ import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.StrUtil;
import com.aizuda.snailjob.common.core.util.JsonUtil;
import com.aizuda.snailjob.common.core.util.StreamUtils;
import com.aizuda.snailjob.server.common.dto.PartitionTask;
import com.aizuda.snailjob.server.common.exception.SnailJobServerException;
import com.aizuda.snailjob.server.common.strategy.WaitStrategies;
import com.aizuda.snailjob.server.common.util.CronUtils;
import com.aizuda.snailjob.server.common.util.PartitionTaskUtils;
import com.aizuda.snailjob.server.web.model.base.PageResult;
import com.aizuda.snailjob.server.web.model.request.ExportSceneVO;
import com.aizuda.snailjob.server.web.model.request.SceneConfigQueryVO;
import com.aizuda.snailjob.server.web.model.request.SceneConfigRequestVO;
import com.aizuda.snailjob.server.web.model.request.UserSessionVO;
@ -29,15 +32,16 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.PageDTO;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.*;
/**
* @author: opensnail
@ -75,9 +79,10 @@ public class SceneConfigServiceImpl implements SceneConfigService {
new LambdaQueryWrapper<RetrySceneConfig>()
.eq(RetrySceneConfig::getNamespaceId, namespaceId)
.in(userSessionVO.isUser(), RetrySceneConfig::getGroupName, userSessionVO.getGroupNames())
.eq(Objects.nonNull(queryVO.getSceneStatus()), RetrySceneConfig::getSceneStatus, queryVO.getSceneStatus())
.eq(StrUtil.isNotBlank(queryVO.getGroupName()),
RetrySceneConfig::getGroupName, StrUtil.trim(queryVO.getGroupName()))
.eq(StrUtil.isNotBlank(queryVO.getSceneName()),
.likeRight(StrUtil.isNotBlank(queryVO.getSceneName()),
RetrySceneConfig::getSceneName, StrUtil.trim(queryVO.getSceneName()))
.orderByDesc(RetrySceneConfig::getCreateDt));
@ -187,18 +192,35 @@ public class SceneConfigServiceImpl implements SceneConfigService {
}
@Override
public String exportSceneConfig(final Set<Long> sceneIds) {
public String exportSceneConfig(final ExportSceneVO exportSceneVO) {
List<SceneConfigRequestVO> requestList = new ArrayList<>();
PartitionTaskUtils.process(startId -> {
List<RetrySceneConfig> sceneConfigs = accessTemplate.getSceneConfigAccess()
.list(new LambdaQueryWrapper<RetrySceneConfig>()
.listPage(new PageDTO<>(0, 500), new LambdaQueryWrapper<RetrySceneConfig>()
.eq(RetrySceneConfig::getNamespaceId, UserSessionUtils.currentUserSession().getNamespaceId())
// TODO 若导出全部需要分页查询，避免一次拉取太多数据
.in(CollUtil.isNotEmpty(sceneIds), RetrySceneConfig::getId, sceneIds)
.eq(Objects.nonNull(exportSceneVO.getSceneStatus()), RetrySceneConfig::getSceneStatus, exportSceneVO.getSceneStatus())
.eq(StrUtil.isNotBlank(exportSceneVO.getGroupName()),
RetrySceneConfig::getGroupName, StrUtil.trim(exportSceneVO.getGroupName()))
.likeRight(StrUtil.isNotBlank(exportSceneVO.getSceneName()),
RetrySceneConfig::getSceneName, StrUtil.trim(exportSceneVO.getSceneName()))
.in(CollUtil.isNotEmpty(exportSceneVO.getSceneIds()), RetrySceneConfig::getId, exportSceneVO.getSceneIds())
.ge(RetrySceneConfig::getId, startId)
.orderByAsc(RetrySceneConfig::getId)
).getRecords();
return StreamUtils.toList(sceneConfigs, SceneConfigPartitionTask::new);
}, partitionTasks -> {
List<SceneConfigPartitionTask> partitionTaskList = (List<SceneConfigPartitionTask>) partitionTasks;
requestList.addAll(SceneConfigConverter.INSTANCE.toSceneConfigRequestVOs(
StreamUtils.toList(partitionTaskList, SceneConfigPartitionTask::getConfig))
);
}, 0);
return JsonUtil.toJsonString(SceneConfigConverter.INSTANCE.toSceneConfigRequestVOs(sceneConfigs));
return JsonUtil.toJsonString(requestList);
}
private void batchSaveSceneConfig(final List<SceneConfigRequestVO> requests, final String namespaceId) {
Set<String> groupNameSet = Sets.newHashSet();
@ -242,11 +264,22 @@ public class SceneConfigServiceImpl implements SceneConfigService {
retrySceneConfig.setTriggerInterval(StrUtil.EMPTY);
}
// TODO 优化成批量插入
Assert.isTrue(1 == sceneConfigAccess.insert(retrySceneConfig),
() -> new SnailJobServerException("failed to insert scene. retrySceneConfig:[{}]",
JsonUtil.toJsonString(retrySceneConfig)));
}
}
@EqualsAndHashCode(callSuper = true)
@Getter
private static class SceneConfigPartitionTask extends PartitionTask {
// 这里就直接放RetrySceneConfig为了后面若加字段不需要再这里在调整了
private final RetrySceneConfig config;
public SceneConfigPartitionTask(@NotNull RetrySceneConfig config) {
this.config = config;
setId(config.getId());
}
}
}

4
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/util/ImportUtils.java
View File

@ -15,6 +15,10 @@ public class ImportUtils {
private static final List<String> FILE_EXTENSIONS = List.of("json");
public static @NotNull <VO> List<VO> parseList(MultipartFile file, Class<VO> clazz) throws IOException {
if (file.isEmpty()) {
throw new SnailJobCommonException("请选择一个文件上传");
}
// 保存文件到服务器
String suffix = FileUtil.getSuffix(file.getOriginalFilename());
if (!FILE_EXTENSIONS.contains(suffix)) {