gtsoft/gtsoft-snail-job-server
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects 1 Releases Wiki Activity

feat(3.2.0): 修复日志合并事务问题

Browse Source
This commit is contained in:
byteblogs168 2024-04-01 18:52:28 +08:00
parent 328d4f0569
commit 6d0c331ffc
5 changed files with 119 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/sql/easy_retry_mysql.sql
View File

@ -159,7 +159,7 @@ CREATE TABLE `retry_task_log_message`
`group_name` varchar(64) NOT NULL COMMENT '组名称', `group_name` varchar(64) NOT NULL COMMENT '组名称',
`unique_id` varchar(64) NOT NULL COMMENT '同组下id唯一', `unique_id` varchar(64) NOT NULL COMMENT '同组下id唯一',
`create_dt` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `create_dt` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`message` text NOT NULL COMMENT '异常信息', `message` longtext NOT NULL COMMENT '异常信息',
`log_num` int(11) NOT NULL DEFAULT 1 COMMENT '日志数量', `log_num` int(11) NOT NULL DEFAULT 1 COMMENT '日志数量',
`real_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上报时间', `real_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上报时间',
`client_info` varchar(128) DEFAULT NULL COMMENT '客户端地址 clientId#ip:port', `client_info` varchar(128) DEFAULT NULL COMMENT '客户端地址 clientId#ip:port',

48
easy-retry-server/easy-retry-server-job-task/src/main/java/com/aizuda/easy/retry/server/job/task/support/schedule/JobLogMergeSchedule.java
View File

@ -22,12 +22,14 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import org.springframework.transaction.support.TransactionTemplate; import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.util.CollectionUtils;
import java.time.Duration; import java.time.Duration;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@ -50,18 +52,13 @@ import static java.util.stream.Collectors.*;
*/ */
@Slf4j @Slf4j
@Component @Component
@RequiredArgsConstructor
public class JobLogMergeSchedule extends AbstractSchedule implements Lifecycle { public class JobLogMergeSchedule extends AbstractSchedule implements Lifecycle {
@Autowired private final SystemProperties systemProperties;
private SystemProperties systemProperties; private final JobTaskBatchMapper jobTaskBatchMapper;
@Autowired private final JobLogMessageMapper jobLogMessageMapper;
private JobTaskBatchMapper jobTaskBatchMapper; private final TransactionTemplate transactionTemplate;
@Autowired
private JobTaskMapper jobTaskMapper;
@Autowired
private JobLogMessageMapper jobLogMessageMapper;
@Autowired
private TransactionTemplate transactionTemplate;
// last merge log time // last merge log time
private static Long lastMergeLogTime = 0L; private static Long lastMergeLogTime = 0L;
@ -127,19 +124,16 @@ public class JobLogMergeSchedule extends AbstractSchedule implements Lifecycle {
*/ */
public void processJobLogPartitionTasks(List<? extends PartitionTask> partitionTasks) { public void processJobLogPartitionTasks(List<? extends PartitionTask> partitionTasks) {
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(final TransactionStatus status) {
// Waiting for merge JobTaskBatchList // Waiting for merge JobTaskBatchList
List<Long> ids = partitionTasks.stream().map(PartitionTask::getId).collect(Collectors.toList()); List<Long> ids = partitionTasks.stream().map(PartitionTask::getId).collect(Collectors.toList());
if (ids == null || ids.isEmpty()) { if (CollectionUtils.isEmpty(ids)) {
return; return;
} }
// Waiting for deletion JobLogMessageList // Waiting for deletion JobLogMessageList
List<JobLogMessage> jobLogMessageList = jobLogMessageMapper.selectList(new LambdaQueryWrapper<JobLogMessage>().in(JobLogMessage::getTaskBatchId, ids)); List<JobLogMessage> jobLogMessageList = jobLogMessageMapper.selectList(
if (jobLogMessageList == null || jobLogMessageList.isEmpty()) { new LambdaQueryWrapper<JobLogMessage>().in(JobLogMessage::getTaskBatchId, ids));
if (CollectionUtils.isEmpty(jobLogMessageList)) {
return; return;
} }
@ -166,16 +160,21 @@ public class JobLogMergeSchedule extends AbstractSchedule implements Lifecycle {
// 500条数据为一个批次 // 500条数据为一个批次
List<List<String>> partitionMessages = Lists.partition(mergeMessages, systemProperties.getMergeLogNum()); List<List<String>> partitionMessages = Lists.partition(mergeMessages, systemProperties.getMergeLogNum());
for (int i = 0; i < partitionMessages.size(); i++) { for (List<String> partitionMessage : partitionMessages) {
// 深拷贝 // 深拷贝
JobLogMessage jobLogMessage = JobTaskConverter.INSTANCE.toJobLogMessage(jobLogMessageMap.getValue().get(0)); JobLogMessage jobLogMessage = JobTaskConverter.INSTANCE.toJobLogMessage(
List<String> messages = partitionMessages.get(i); jobLogMessageMap.getValue().get(0));
jobLogMessage.setLogNum(messages.size()); jobLogMessage.setLogNum(partitionMessage.size());
jobLogMessage.setMessage(JsonUtil.toJsonString(messages)); jobLogMessage.setMessage(JsonUtil.toJsonString(partitionMessage));
jobLogMessageInsertBatchIds.add(jobLogMessage); jobLogMessageInsertBatchIds.add(jobLogMessage);
} }
transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(final TransactionStatus status) {
// 批量删除、更新日志 // 批量删除、更新日志
if (CollectionUtil.isNotEmpty(jobLogMessageDeleteBatchIds)) { if (CollectionUtil.isNotEmpty(jobLogMessageDeleteBatchIds)) {
jobLogMessageMapper.deleteBatchIds(jobLogMessageDeleteBatchIds); jobLogMessageMapper.deleteBatchIds(jobLogMessageDeleteBatchIds);
@ -184,10 +183,11 @@ public class JobLogMergeSchedule extends AbstractSchedule implements Lifecycle {
jobLogMessageMapper.batchInsert(jobLogMessageInsertBatchIds); jobLogMessageMapper.batchInsert(jobLogMessageInsertBatchIds);
} }
} }
}
}); });
} }
}
@Override @Override
public void start() { public void start() {
taskScheduler.scheduleAtFixedRate(this::execute, Duration.parse("PT1H")); taskScheduler.scheduleAtFixedRate(this::execute, Duration.parse("PT1H"));

12
easy-retry-server/easy-retry-server-job-task/src/main/java/com/aizuda/easy/retry/server/job/task/support/schedule/WorkflowJobSummarySchedule.java
View File

@ -16,6 +16,7 @@ import com.aizuda.easy.retry.template.datasource.persistence.po.JobSummary;
import com.aizuda.easy.retry.template.datasource.persistence.po.WorkflowTaskBatch; import com.aizuda.easy.retry.template.datasource.persistence.po.WorkflowTaskBatch;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@ -40,14 +41,11 @@ import java.util.stream.Collectors;
*/ */
@Component @Component
@Slf4j @Slf4j
@RequiredArgsConstructor
public class WorkflowJobSummarySchedule extends AbstractSchedule implements Lifecycle { public class WorkflowJobSummarySchedule extends AbstractSchedule implements Lifecycle {
private final JobTaskBatchMapper jobTaskBatchMapper;
@Autowired private final JobSummaryMapper jobSummaryMapper;
private JobTaskBatchMapper jobTaskBatchMapper; private final SystemProperties systemProperties;
@Autowired
private JobSummaryMapper jobSummaryMapper;
@Autowired
private SystemProperties systemProperties;
@Override @Override
public String lockName() { public String lockName() {

2
easy-retry-server/easy-retry-server-retry-task/src/main/java/com/aizuda/easy/retry/server/retry/task/client/RetryRpcClient.java
View File

@ -32,7 +32,7 @@ public interface RetryRpcClient {
@Mapping(path = "/retry/generate/idempotent-id/v1", method = RequestMethod.POST) @Mapping(path = "/retry/generate/idempotent-id/v1", method = RequestMethod.POST)
Result generateIdempotentId(@Body GenerateRetryIdempotentIdDTO retryCallbackDTO); Result generateIdempotentId(@Body GenerateRetryIdempotentIdDTO retryCallbackDTO);
@Mapping(path = "/retry/sync/version/v1") @Mapping(path = "/retry/sync/version/v1", method = RequestMethod.POST)
Result syncConfig(@Body ConfigDTO configDTO); Result syncConfig(@Body ConfigDTO configDTO);
} }

20
easy-retry-server/easy-retry-server-retry-task/src/main/java/com/aizuda/easy/retry/server/retry/task/support/schedule/RetryLogMergeSchedule.java
View File

@ -29,9 +29,10 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import org.springframework.transaction.support.TransactionTemplate; import org.springframework.transaction.support.TransactionTemplate;
@ -90,7 +91,7 @@ public class RetryLogMergeSchedule extends AbstractSchedule implements Lifecycle
// merge job log // merge job log
long total; long total;
LocalDateTime endTime = LocalDateTime.now().minusDays(systemProperties.getMergeLogDays()); LocalDateTime endTime = LocalDateTime.now().minusDays(systemProperties.getMergeLogDays());
total = PartitionTaskUtils.process(startId -> jobTaskBatchList(startId, endTime), total = PartitionTaskUtils.process(startId -> retryLogList(startId, endTime),
this::processJobLogPartitionTasks, 0); this::processJobLogPartitionTasks, 0);
EasyRetryLog.LOCAL.debug("job merge success total:[{}]", total); EasyRetryLog.LOCAL.debug("job merge success total:[{}]", total);
@ -109,7 +110,7 @@ public class RetryLogMergeSchedule extends AbstractSchedule implements Lifecycle
* @param endTime * @param endTime
* @return * @return
*/ */
private List<RetryMergePartitionTaskDTO> jobTaskBatchList(Long startId, LocalDateTime endTime) { private List<RetryMergePartitionTaskDTO> retryLogList(Long startId, LocalDateTime endTime) {
List<RetryTaskLog> jobTaskBatchList = retryTaskLogMapper.selectPage( List<RetryTaskLog> jobTaskBatchList = retryTaskLogMapper.selectPage(
new Page<>(0, 1000), new Page<>(0, 1000),
@ -129,10 +130,6 @@ public class RetryLogMergeSchedule extends AbstractSchedule implements Lifecycle
*/ */
public void processJobLogPartitionTasks(List<? extends PartitionTask> partitionTasks) { public void processJobLogPartitionTasks(List<? extends PartitionTask> partitionTasks) {
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(final TransactionStatus status) {
// Waiting for merge RetryTaskLog // Waiting for merge RetryTaskLog
List<String> ids = partitionTasks.stream().map(PartitionTask::getUniqueId).collect(Collectors.toList()); List<String> ids = partitionTasks.stream().map(PartitionTask::getUniqueId).collect(Collectors.toList());
if (CollectionUtil.isEmpty(ids)) { if (CollectionUtil.isEmpty(ids)) {
@ -185,6 +182,10 @@ public class RetryLogMergeSchedule extends AbstractSchedule implements Lifecycle
jobLogMessageUpdateList.add(jobLogMessage); jobLogMessageUpdateList.add(jobLogMessage);
} }
transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(final TransactionStatus status) {
// 批量删除、更新日志 // 批量删除、更新日志
if (CollectionUtil.isNotEmpty(jobLogMessageDeleteBatchIds)) { if (CollectionUtil.isNotEmpty(jobLogMessageDeleteBatchIds)) {
retryTaskLogMessageMapper.deleteBatchIds(jobLogMessageDeleteBatchIds); retryTaskLogMessageMapper.deleteBatchIds(jobLogMessageDeleteBatchIds);
@ -193,11 +194,12 @@ public class RetryLogMergeSchedule extends AbstractSchedule implements Lifecycle
retryTaskLogMessageMapper.batchInsert(jobLogMessageUpdateList); retryTaskLogMessageMapper.batchInsert(jobLogMessageUpdateList);
} }
} }
}
}); });
} }
}
@Override @Override
public void start() { public void start() {
taskScheduler.scheduleAtFixedRate(this::execute, Duration.parse("PT1H")); taskScheduler.scheduleAtFixedRate(this::execute, Duration.parse("PT1H"));