gtsoft/gtsoft-snail-job-server
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects 1 Releases Wiki Activity

feat(1.5.0-beta1): 重试任务支持防止任务重复拉取配置

Browse Source
This commit is contained in:
opensnail 2025-04-06 08:45:52 +08:00
parent 34e826cd97
commit 4de55b5fd3
3 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
snail-job-server/snail-job-server-common/src/main/java/com/aizuda/snailjob/server/common/dto/ScanTask.java
View File

@ -14,11 +14,8 @@ import java.util.Set;
@Data
public class ScanTask {
// private String namespaceId;
//
// private String groupName;
private Set<Integer> buckets;
// private Integer groupPartition;
private String bucketStr;
}

4
snail-job-server/snail-job-server-retry-task/src/main/java/com/aizuda/snailjob/server/retry/task/support/dispatch/ScanRetryActor.java
View File

@ -38,6 +38,7 @@ import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.stream.Collectors;
/**
@ -53,6 +54,7 @@ import java.util.stream.Collectors;
@Slf4j
@RequiredArgsConstructor
public class ScanRetryActor extends AbstractActor {
public static final ConcurrentSkipListSet<String> REPEATED_PULL= new ConcurrentSkipListSet<>(new ArrayList<>());
private final SystemProperties systemProperties;
private final AccessTemplate accessTemplate;
private final RetryMapper retryMapper;
@ -67,6 +69,8 @@ public class ScanRetryActor extends AbstractActor {
doScan(config);
} catch (Exception e) {
SnailJobLog.LOCAL.error("Data scanner processing exception. [{}]", config, e);
} finally {
REPEATED_PULL.remove(config.getBucketStr());
}
}).build();

9
snail-job-server/snail-job-server-starter/src/main/java/com/aizuda/snailjob/server/starter/dispatch/ConsumerBucketActor.java
View File

@ -1,5 +1,7 @@
package com.aizuda.snailjob.server.starter.dispatch;
import cn.hutool.core.util.StrUtil;
import com.aizuda.snailjob.server.retry.task.support.dispatch.ScanRetryActor;
import org.apache.pekko.actor.AbstractActor;
import org.apache.pekko.actor.ActorRef;
import cn.hutool.core.collection.CollUtil;
@ -71,10 +73,17 @@ public class ConsumerBucketActor extends AbstractActor {
List<List<Integer>> partitions = Lists.partition(new ArrayList<>(totalBuckets),
(totalBuckets.size() + retryMaxPullParallel - 1) / retryMaxPullParallel);
for (List<Integer> buckets : partitions) {
String key = StrUtil.join(StrUtil.COMMA, new TreeSet<>(buckets));
if (ScanRetryActor.REPEATED_PULL.contains(key)) {
SnailJobLog.LOCAL.warn("Discard the current scanning task because there are ongoing tasks in the current batch.[{}]", key);
continue;
}
ScanTask scanTask = new ScanTask();
scanTask.setBucketStr(key);
scanTask.setBuckets(new HashSet<>(buckets));
ActorRef scanRetryActorRef = ActorGenerator.scanRetryActor();
scanRetryActorRef.tell(scanTask, scanRetryActorRef);
ScanRetryActor.REPEATED_PULL.add(key);
}
}