feat(sj_1.0.0): 优化重试场景的导出

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/controller/GroupConfigController.java

@@ -93,10 +93,6 @@ public class GroupConfigController {
     @PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
     @LoginRequired(role = RoleEnum.ADMIN)
     public void importScene(@RequestPart("file") MultipartFile file) throws IOException {
-        if (file.isEmpty()) {
-            throw new SnailJobCommonException("请选择一个文件上传");
-        }
-
         groupConfigService.importGroup(ImportUtils.parseList(file, GroupConfigRequestVO.class));
     }
 

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/controller/JobController.java

@@ -101,21 +101,16 @@ public class JobController {
     }
 
     @PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
-    @LoginRequired(role = RoleEnum.ADMIN)
+    @LoginRequired
     public void importScene(@RequestPart("file") MultipartFile file) throws IOException {
-        if (file.isEmpty()) {
+        jobService.importJobs(ImportUtils.parseList(file, JobRequestVO.class));
-            throw new SnailJobCommonException("请选择一个文件上传");
-        }
-        List<JobRequestVO> requestList = ImportUtils.parseList(file, JobRequestVO.class);
-        jobService.importJobs(requestList);
     }
 
     @PostMapping("/export")
-    @LoginRequired(role = RoleEnum.ADMIN)
+    @LoginRequired
     @OriginalControllerReturnValue
     public ResponseEntity<String> exportGroup(@RequestBody Set<Long> jobIds) {
-        String jobsJson = jobService.exportJobs(jobIds);
+        return ExportUtils.doExport(jobService.exportJobs(jobIds));
-        return ExportUtils.doExport(jobsJson);
     }
 
 }

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/controller/SceneConfigController.java

@@ -4,6 +4,7 @@ import com.aizuda.snailjob.common.core.annotation.OriginalControllerReturnValue;
 import com.aizuda.snailjob.common.core.exception.SnailJobCommonException;
 import com.aizuda.snailjob.server.web.annotation.LoginRequired;
 import com.aizuda.snailjob.server.web.model.base.PageResult;
+import com.aizuda.snailjob.server.web.model.request.ExportSceneVO;
 import com.aizuda.snailjob.server.web.model.request.SceneConfigQueryVO;
 import com.aizuda.snailjob.server.web.model.request.SceneConfigRequestVO;
 import com.aizuda.snailjob.server.web.model.response.SceneConfigResponseVO;
@@ -72,10 +73,6 @@ public class SceneConfigController {
     @LoginRequired
     @PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
     public void importScene(@RequestPart("file") MultipartFile file) throws IOException {
-        if (file.isEmpty()) {
-            throw new SnailJobCommonException("请选择一个文件上传");
-        }
-
         // 写入数据
         sceneConfigService.importSceneConfig(ImportUtils.parseList(file, SceneConfigRequestVO.class));
     }
@@ -83,9 +80,8 @@ public class SceneConfigController {
     @LoginRequired
     @PostMapping("/export")
     @OriginalControllerReturnValue
-    public ResponseEntity<String> export(@RequestBody Set<Long> sceneIds) {
+    public ResponseEntity<String> export(@RequestBody ExportSceneVO exportSceneVO) {
-        return ExportUtils.doExport(sceneConfigService.exportSceneConfig(sceneIds));
+        return ExportUtils.doExport(sceneConfigService.exportSceneConfig(exportSceneVO));
-
     }
 
 }

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/request/ExportSceneVO.java

@@ -0,0 +1,23 @@
+package com.aizuda.snailjob.server.web.model.request;
+
+import lombok.Data;
+
+import java.util.Set;
+
+/**
+ * @author: opensnail
+ * @date : 2024-05-29
+ * @since : sj_1.0.0
+ */
+@Data
+public class ExportSceneVO {
+
+    private String groupName;
+
+    private Integer sceneStatus;
+
+    private String sceneName;
+
+    private Set<Long> sceneIds;
+
+}

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/request/SceneConfigQueryVO.java

@@ -14,4 +14,5 @@ import lombok.EqualsAndHashCode;
 public class SceneConfigQueryVO extends BaseQueryVO {
     private String groupName;
     private String sceneName;
+    private Integer sceneStatus;
 }

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/SceneConfigService.java

@@ -1,6 +1,7 @@
 package com.aizuda.snailjob.server.web.service;
 
 import com.aizuda.snailjob.server.web.model.base.PageResult;
+import com.aizuda.snailjob.server.web.model.request.ExportSceneVO;
 import com.aizuda.snailjob.server.web.model.request.SceneConfigQueryVO;
 import com.aizuda.snailjob.server.web.model.request.SceneConfigRequestVO;
 import com.aizuda.snailjob.server.web.model.response.SceneConfigResponseVO;
@@ -8,7 +9,6 @@ import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotEmpty;
 
 import java.util.List;
-import java.util.Set;
 
 /**
  * @author: opensnail
@@ -30,6 +30,6 @@ public interface SceneConfigService {
 
     void importSceneConfig(@Valid @NotEmpty(message = "导入数据不能为空") List<SceneConfigRequestVO> requests);
 
-    String exportSceneConfig(Set<Long> sceneIds);
+    String exportSceneConfig(ExportSceneVO exportSceneVO);
 
 }

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/impl/GroupConfigServiceImpl.java

@@ -439,8 +439,7 @@ public class GroupConfigServiceImpl implements GroupConfigService {
             return groupConfigs.stream().map(GroupConfigPartitionTask::new).toList();
         }), partitionTasks -> {
             List<GroupConfigPartitionTask> configPartitionTasks = (List<GroupConfigPartitionTask>) partitionTasks;
-            List<GroupConfig> configs = configPartitionTasks.stream().map(GroupConfigPartitionTask::getConfig)
+            List<GroupConfig> configs = StreamUtils.toList(configPartitionTasks, GroupConfigPartitionTask::getConfig);
-                .collect(Collectors.toList());
             allRequestList.addAll(GroupConfigConverter.INSTANCE.toGroupConfigRequestVOs(configs));
         }, 0);
 

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/impl/SceneConfigServiceImpl.java

@@ -5,10 +5,13 @@ import cn.hutool.core.lang.Assert;
 import cn.hutool.core.util.StrUtil;
 import com.aizuda.snailjob.common.core.util.JsonUtil;
 import com.aizuda.snailjob.common.core.util.StreamUtils;
+import com.aizuda.snailjob.server.common.dto.PartitionTask;
 import com.aizuda.snailjob.server.common.exception.SnailJobServerException;
 import com.aizuda.snailjob.server.common.strategy.WaitStrategies;
 import com.aizuda.snailjob.server.common.util.CronUtils;
+import com.aizuda.snailjob.server.common.util.PartitionTaskUtils;
 import com.aizuda.snailjob.server.web.model.base.PageResult;
+import com.aizuda.snailjob.server.web.model.request.ExportSceneVO;
 import com.aizuda.snailjob.server.web.model.request.SceneConfigQueryVO;
 import com.aizuda.snailjob.server.web.model.request.SceneConfigRequestVO;
 import com.aizuda.snailjob.server.web.model.request.UserSessionVO;
@@ -29,15 +32,16 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.PageDTO;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Sets.SetView;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.validation.annotation.Validated;
 
 import java.time.LocalDateTime;
-import java.util.List;
+import java.util.*;
-import java.util.Optional;
-import java.util.Set;
 
 /**
  * @author: opensnail
@@ -53,8 +57,8 @@ public class SceneConfigServiceImpl implements SceneConfigService {
 
     private static void checkExecuteInterval(SceneConfigRequestVO requestVO) {
         if (Lists.newArrayList(WaitStrategies.WaitStrategyEnum.FIXED.getType(),
-                WaitStrategies.WaitStrategyEnum.RANDOM.getType())
+                        WaitStrategies.WaitStrategyEnum.RANDOM.getType())
-            .contains(requestVO.getBackOff())) {
+                .contains(requestVO.getBackOff())) {
             if (Integer.parseInt(requestVO.getTriggerInterval()) < 10) {
                 throw new SnailJobServerException("间隔时间不得小于10");
             }
@@ -72,14 +76,15 @@ public class SceneConfigServiceImpl implements SceneConfigService {
         UserSessionVO userSessionVO = UserSessionUtils.currentUserSession();
         String namespaceId = userSessionVO.getNamespaceId();
         pageDTO = accessTemplate.getSceneConfigAccess().listPage(pageDTO,
-            new LambdaQueryWrapper<RetrySceneConfig>()
+                new LambdaQueryWrapper<RetrySceneConfig>()
-                .eq(RetrySceneConfig::getNamespaceId, namespaceId)
+                        .eq(RetrySceneConfig::getNamespaceId, namespaceId)
-                .in(userSessionVO.isUser(), RetrySceneConfig::getGroupName, userSessionVO.getGroupNames())
+                        .in(userSessionVO.isUser(), RetrySceneConfig::getGroupName, userSessionVO.getGroupNames())
-                .eq(StrUtil.isNotBlank(queryVO.getGroupName()),
+                        .eq(Objects.nonNull(queryVO.getSceneStatus()), RetrySceneConfig::getSceneStatus, queryVO.getSceneStatus())
-                    RetrySceneConfig::getGroupName, StrUtil.trim(queryVO.getGroupName()))
+                        .eq(StrUtil.isNotBlank(queryVO.getGroupName()),
-                .eq(StrUtil.isNotBlank(queryVO.getSceneName()),
+                                RetrySceneConfig::getGroupName, StrUtil.trim(queryVO.getGroupName()))
-                    RetrySceneConfig::getSceneName, StrUtil.trim(queryVO.getSceneName()))
+                        .likeRight(StrUtil.isNotBlank(queryVO.getSceneName()),
-                .orderByDesc(RetrySceneConfig::getCreateDt));
+                                RetrySceneConfig::getSceneName, StrUtil.trim(queryVO.getSceneName()))
+                        .orderByDesc(RetrySceneConfig::getCreateDt));
 
         return new PageResult<>(pageDTO, SceneConfigResponseVOConverter.INSTANCE.convertList(pageDTO.getRecords()));
 
@@ -91,12 +96,12 @@ public class SceneConfigServiceImpl implements SceneConfigService {
         String namespaceId = UserSessionUtils.currentUserSession().getNamespaceId();
 
         List<RetrySceneConfig> retrySceneConfigs = accessTemplate.getSceneConfigAccess()
-            .list(new LambdaQueryWrapper<RetrySceneConfig>()
+                .list(new LambdaQueryWrapper<RetrySceneConfig>()
-                .eq(RetrySceneConfig::getNamespaceId, namespaceId)
+                        .eq(RetrySceneConfig::getNamespaceId, namespaceId)
-                .eq(RetrySceneConfig::getGroupName, groupName)
+                        .eq(RetrySceneConfig::getGroupName, groupName)
-                .select(RetrySceneConfig::getSceneName,
+                        .select(RetrySceneConfig::getSceneName,
-                    RetrySceneConfig::getDescription, RetrySceneConfig::getMaxRetryCount)
+                                RetrySceneConfig::getDescription, RetrySceneConfig::getMaxRetryCount)
-                .orderByDesc(RetrySceneConfig::getCreateDt));
+                        .orderByDesc(RetrySceneConfig::getCreateDt));
 
         return SceneConfigResponseVOConverter.INSTANCE.convertList(retrySceneConfigs);
     }
@@ -108,10 +113,10 @@ public class SceneConfigServiceImpl implements SceneConfigService {
         String namespaceId = UserSessionUtils.currentUserSession().getNamespaceId();
         ConfigAccess<RetrySceneConfig> sceneConfigAccess = accessTemplate.getSceneConfigAccess();
         Assert.isTrue(0 == sceneConfigAccess.count(
-            new LambdaQueryWrapper<RetrySceneConfig>()
+                new LambdaQueryWrapper<RetrySceneConfig>()
-                .eq(RetrySceneConfig::getNamespaceId, namespaceId)
+                        .eq(RetrySceneConfig::getNamespaceId, namespaceId)
-                .eq(RetrySceneConfig::getGroupName, requestVO.getGroupName())
+                        .eq(RetrySceneConfig::getGroupName, requestVO.getGroupName())
-                .eq(RetrySceneConfig::getSceneName, requestVO.getSceneName())
+                        .eq(RetrySceneConfig::getSceneName, requestVO.getSceneName())
 
         ), () -> new SnailJobServerException("场景名称重复. {}", requestVO.getSceneName()));
 
@@ -123,8 +128,8 @@ public class SceneConfigServiceImpl implements SceneConfigService {
         }
 
         Assert.isTrue(1 == sceneConfigAccess.insert(retrySceneConfig),
-            () -> new SnailJobServerException("failed to insert scene. retrySceneConfig:[{}]",
+                () -> new SnailJobServerException("failed to insert scene. retrySceneConfig:[{}]",
-                JsonUtil.toJsonString(retrySceneConfig)));
+                        JsonUtil.toJsonString(retrySceneConfig)));
 
         // 同步配置到客户端
         SyncConfigHandler.addSyncTask(requestVO.getGroupName(), namespaceId);
@@ -144,14 +149,14 @@ public class SceneConfigServiceImpl implements SceneConfigService {
         String namespaceId = UserSessionUtils.currentUserSession().getNamespaceId();
 
         retrySceneConfig.setTriggerInterval(
-            Optional.ofNullable(retrySceneConfig.getTriggerInterval()).orElse(StrUtil.EMPTY));
+                Optional.ofNullable(retrySceneConfig.getTriggerInterval()).orElse(StrUtil.EMPTY));
         Assert.isTrue(1 == accessTemplate.getSceneConfigAccess().update(retrySceneConfig,
-                new LambdaUpdateWrapper<RetrySceneConfig>()
+                        new LambdaUpdateWrapper<RetrySceneConfig>()
-                    .eq(RetrySceneConfig::getNamespaceId, namespaceId)
+                                .eq(RetrySceneConfig::getNamespaceId, namespaceId)
-                    .eq(RetrySceneConfig::getGroupName, requestVO.getGroupName())
+                                .eq(RetrySceneConfig::getGroupName, requestVO.getGroupName())
-                    .eq(RetrySceneConfig::getSceneName, requestVO.getSceneName())),
+                                .eq(RetrySceneConfig::getSceneName, requestVO.getSceneName())),
-            () -> new SnailJobServerException("failed to update scene. retrySceneConfig:[{}]",
+                () -> new SnailJobServerException("failed to update scene. retrySceneConfig:[{}]",
-                JsonUtil.toJsonString(retrySceneConfig)));
+                        JsonUtil.toJsonString(retrySceneConfig)));
 
         // 同步配置到客户端
         SyncConfigHandler.addSyncTask(requestVO.getGroupName(), namespaceId);
@@ -161,8 +166,8 @@ public class SceneConfigServiceImpl implements SceneConfigService {
     @Override
     public SceneConfigResponseVO getSceneConfigDetail(Long id) {
         RetrySceneConfig retrySceneConfig = accessTemplate.getSceneConfigAccess()
-            .one(new LambdaQueryWrapper<RetrySceneConfig>()
+                .one(new LambdaQueryWrapper<RetrySceneConfig>()
-                .eq(RetrySceneConfig::getId, id));
+                        .eq(RetrySceneConfig::getId, id));
         return SceneConfigResponseVOConverter.INSTANCE.convert(retrySceneConfig);
     }
 
@@ -175,9 +180,9 @@ public class SceneConfigServiceImpl implements SceneConfigService {
         config.setSceneStatus(status);
 
         return 1 == accessTemplate.getSceneConfigAccess().update(config,
-            new LambdaUpdateWrapper<RetrySceneConfig>()
+                new LambdaUpdateWrapper<RetrySceneConfig>()
-                .eq(RetrySceneConfig::getId, id)
+                        .eq(RetrySceneConfig::getId, id)
-                .eq(RetrySceneConfig::getNamespaceId, namespaceId));
+                        .eq(RetrySceneConfig::getNamespaceId, namespaceId));
     }
 
     @Override
@@ -187,18 +192,35 @@ public class SceneConfigServiceImpl implements SceneConfigService {
     }
 
     @Override
-    public String exportSceneConfig(final Set<Long> sceneIds) {
+    public String exportSceneConfig(final ExportSceneVO exportSceneVO) {
-        List<RetrySceneConfig> sceneConfigs = accessTemplate.getSceneConfigAccess()
+
-            .list(new LambdaQueryWrapper<RetrySceneConfig>()
+        List<SceneConfigRequestVO> requestList = new ArrayList<>();
-                .eq(RetrySceneConfig::getNamespaceId, UserSessionUtils.currentUserSession().getNamespaceId())
+
-                // TODO 若导出全部需要分页查询，避免一次拉取太多数据
+        PartitionTaskUtils.process(startId -> {
-                .in(CollUtil.isNotEmpty(sceneIds), RetrySceneConfig::getId, sceneIds)
+            List<RetrySceneConfig> sceneConfigs = accessTemplate.getSceneConfigAccess()
+                    .listPage(new PageDTO<>(0, 500), new LambdaQueryWrapper<RetrySceneConfig>()
+                            .eq(RetrySceneConfig::getNamespaceId, UserSessionUtils.currentUserSession().getNamespaceId())
+                            .eq(Objects.nonNull(exportSceneVO.getSceneStatus()), RetrySceneConfig::getSceneStatus, exportSceneVO.getSceneStatus())
+                            .eq(StrUtil.isNotBlank(exportSceneVO.getGroupName()),
+                                    RetrySceneConfig::getGroupName, StrUtil.trim(exportSceneVO.getGroupName()))
+                            .likeRight(StrUtil.isNotBlank(exportSceneVO.getSceneName()),
+                                    RetrySceneConfig::getSceneName, StrUtil.trim(exportSceneVO.getSceneName()))
+                            .in(CollUtil.isNotEmpty(exportSceneVO.getSceneIds()), RetrySceneConfig::getId, exportSceneVO.getSceneIds())
+                            .ge(RetrySceneConfig::getId, startId)
+                            .orderByAsc(RetrySceneConfig::getId)
+                    ).getRecords();
+
+            return StreamUtils.toList(sceneConfigs, SceneConfigPartitionTask::new);
+        }, partitionTasks -> {
+            List<SceneConfigPartitionTask> partitionTaskList = (List<SceneConfigPartitionTask>) partitionTasks;
+            requestList.addAll(SceneConfigConverter.INSTANCE.toSceneConfigRequestVOs(
+                    StreamUtils.toList(partitionTaskList, SceneConfigPartitionTask::getConfig))
             );
+        }, 0);
 
-        return JsonUtil.toJsonString(SceneConfigConverter.INSTANCE.toSceneConfigRequestVOs(sceneConfigs));
+        return JsonUtil.toJsonString(requestList);
     }
 
-
     private void batchSaveSceneConfig(final List<SceneConfigRequestVO> requests, final String namespaceId) {
 
         Set<String> groupNameSet = Sets.newHashSet();
@@ -210,28 +232,28 @@ public class SceneConfigServiceImpl implements SceneConfigService {
         }
 
         List<GroupConfig> groupConfigs = accessTemplate.getGroupConfigAccess()
-            .list(new LambdaQueryWrapper<GroupConfig>()
+                .list(new LambdaQueryWrapper<GroupConfig>()
-                .select(GroupConfig::getGroupName)
+                        .select(GroupConfig::getGroupName)
-                .eq(GroupConfig::getNamespaceId, namespaceId)
+                        .eq(GroupConfig::getNamespaceId, namespaceId)
-                .in(GroupConfig::getGroupName, groupNameSet)
+                        .in(GroupConfig::getGroupName, groupNameSet)
-            );
+                );
 
         SetView<String> notExistedGroupNameSet = Sets.difference(groupNameSet,
-            StreamUtils.toSet(groupConfigs, GroupConfig::getGroupName));
+                StreamUtils.toSet(groupConfigs, GroupConfig::getGroupName));
 
         Assert.isTrue(CollUtil.isEmpty(notExistedGroupNameSet),
-            () -> new SnailJobServerException("导入失败. 原因: 组{}不存在", notExistedGroupNameSet));
+                () -> new SnailJobServerException("导入失败. 原因: 组{}不存在", notExistedGroupNameSet));
 
         ConfigAccess<RetrySceneConfig> sceneConfigAccess = accessTemplate.getSceneConfigAccess();
         List<RetrySceneConfig> sceneConfigs = sceneConfigAccess.list(
-            new LambdaQueryWrapper<RetrySceneConfig>()
+                new LambdaQueryWrapper<RetrySceneConfig>()
-                .select(RetrySceneConfig::getSceneName)
+                        .select(RetrySceneConfig::getSceneName)
-                .eq(RetrySceneConfig::getNamespaceId, namespaceId)
+                        .eq(RetrySceneConfig::getNamespaceId, namespaceId)
-                .in(RetrySceneConfig::getGroupName, groupNameSet)
+                        .in(RetrySceneConfig::getGroupName, groupNameSet)
-                .in(RetrySceneConfig::getSceneName, sceneNameSet));
+                        .in(RetrySceneConfig::getSceneName, sceneNameSet));
 
         Assert.isTrue(CollUtil.isEmpty(sceneConfigs), () -> new SnailJobServerException("导入失败. 原因:场景{}已存在",
-            StreamUtils.toSet(sceneConfigs, RetrySceneConfig::getSceneName)));
+                StreamUtils.toSet(sceneConfigs, RetrySceneConfig::getSceneName)));
 
         LocalDateTime now = LocalDateTime.now();
         List<RetrySceneConfig> retrySceneConfigs = SceneConfigConverter.INSTANCE.toRetrySceneConfigs(requests);
@@ -242,11 +264,22 @@ public class SceneConfigServiceImpl implements SceneConfigService {
                 retrySceneConfig.setTriggerInterval(StrUtil.EMPTY);
             }
 
-            // TODO 优化成批量插入
             Assert.isTrue(1 == sceneConfigAccess.insert(retrySceneConfig),
-                () -> new SnailJobServerException("failed to insert scene. retrySceneConfig:[{}]",
+                    () -> new SnailJobServerException("failed to insert scene. retrySceneConfig:[{}]",
-                    JsonUtil.toJsonString(retrySceneConfig)));
+                            JsonUtil.toJsonString(retrySceneConfig)));
         }
 
     }
+
+    @EqualsAndHashCode(callSuper = true)
+    @Getter
+    private static class SceneConfigPartitionTask extends PartitionTask {
+        // 这里就直接放RetrySceneConfig为了后面若加字段不需要再这里在调整了
+        private final RetrySceneConfig config;
+
+        public SceneConfigPartitionTask(@NotNull RetrySceneConfig config) {
+            this.config = config;
+            setId(config.getId());
+        }
+    }
 }

snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/util/ImportUtils.java

@@ -15,6 +15,10 @@ public class ImportUtils {
     private static final List<String> FILE_EXTENSIONS = List.of("json");
 
     public static @NotNull <VO> List<VO> parseList(MultipartFile file, Class<VO> clazz) throws IOException {
+        if (file.isEmpty()) {
+            throw new SnailJobCommonException("请选择一个文件上传");
+        }
+
         // 保存文件到服务器
         String suffix = FileUtil.getSuffix(file.getOriginalFilename());
         if (!FILE_EXTENSIONS.contains(suffix)) {