!84 feat(sj_1.2.0-beta1):新增CMD、PowerShell、Shell、Http相关执行器

* feat(sj_1.2.0-beta1):新增CMD、PowerShell、Shell、Http相关执行器
This commit is contained in:
Srzou 2024-09-01 03:54:04 +00:00 committed by opensnail
parent a9f387b1d7
commit 18b7b8fc4e
15 changed files with 700 additions and 0 deletions

View File

@ -77,6 +77,11 @@ public class SnailJobProperties {
@NestedConfigurationProperty
private SnailJobMailProperties mail = new SnailJobMailProperties();
/**
* 客户端脚本存储位置
*/
private String workspace;
@Data
public static class ServerConfig {
/**

View File

@ -0,0 +1,148 @@
package com.aizuda.snailjob.client.job.core.executor;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.aizuda.snailjob.client.common.config.SnailJobProperties;
import com.aizuda.snailjob.client.model.ExecuteResult;
import com.aizuda.snailjob.common.core.context.SnailSpringContext;
import com.aizuda.snailjob.common.core.exception.SnailJobInnerExecutorException;
import com.aizuda.snailjob.common.core.util.JsonUtil;
import com.aizuda.snailjob.common.log.SnailJobLog;
import lombok.Data;
import org.springframework.util.StringUtils;
import java.util.Map;
public abstract class AbstractHttpExecutor {
private static final int DEFAULT_TIMEOUT = 60;
public static final SnailJobProperties snailJobProperties = SnailSpringContext.getBean(SnailJobProperties.class);
private static final String DEFAULT_REQUEST_METHOD = "GET";
private static final String POST_REQUEST_METHOD = "POST";
private static final String PUT_REQUEST_METHOD = "PUT";
private static final String DELETE_REQUEST_METHOD = "DELETE";
private static final String HTTP = "http";
private static final String HTTP_PREFIX = "http://";
private static final int HTTP_SUCCESS_CODE = 200;
public ExecuteResult process(HttpParams httpParams) {
if (httpParams == null) {
String message = "HttpParams is null. Verify jobParam configuration.";
logWarn(message);
return ExecuteResult.failure(message);
}
// 校验url
validateAndSetUrl(httpParams);
// 设置默认Method及body
setDefaultMethodAndBody(httpParams);
setDefaultMediaType(httpParams);
setDefaultTimeout(httpParams);
logInfo("Request URL: {}\nUsing request method: {}\nRequest timeout: {} seconds",
httpParams.getUrl(),
httpParams.getMethod(),
httpParams.getTimeout());
HttpRequest httpRequest = buildhutoolRequest(httpParams);
return executeRequestAndHandleResponse(httpRequest);
}
private ExecuteResult executeRequestAndHandleResponse(HttpRequest httpRequest) {
try (HttpResponse response = httpRequest.execute()) {
int errCode = response.getStatus();
String body = response.body();
if (errCode != HTTP_SUCCESS_CODE) {
SnailJobLog.LOCAL.error("{} request to URL: {} failed with code: {}, response body: {}",
httpRequest.getMethod(), httpRequest.getUrl(), errCode, body);
return ExecuteResult.failure("HTTP request failed");
}
return ExecuteResult.success(body);
} catch (Exception e) {
throw new SnailJobInnerExecutorException("[snail-job] HTTP internal executor failed", e);
}
}
private void validateAndSetUrl(HttpParams httpParams) {
if (StringUtils.isEmpty(httpParams.getUrl())) {
throw new SnailJobInnerExecutorException("URL cannot be empty.");
}
httpParams.setUrl(httpParams.getUrl().startsWith(HTTP) ? httpParams.getUrl() : HTTP_PREFIX + httpParams.getUrl());
}
private void setDefaultMethodAndBody(HttpParams httpParams) {
if (StringUtils.isEmpty(httpParams.getMethod())) {
httpParams.setMethod(DEFAULT_REQUEST_METHOD);
} else {
httpParams.setMethod(httpParams.getMethod().toUpperCase());
}
if (!DEFAULT_REQUEST_METHOD.equals(httpParams.getMethod()) && StringUtils.isEmpty(httpParams.getBody())) {
httpParams.setBody(JsonUtil.toJSONString());
logWarn("Using default request body: {}", httpParams.getBody());
}
}
private void setDefaultMediaType(HttpParams httpParams) {
if (!DEFAULT_REQUEST_METHOD.equals(httpParams.getMethod()) && JsonUtil.isValidJson(httpParams.getBody()) && StringUtils.isEmpty(httpParams.getMediaType())) {
httpParams.setMediaType("application/json");
logWarn("Using 'application/json' as media type");
}
}
private void setDefaultTimeout(HttpParams httpParams) {
// 使用milliseconds
httpParams.setTimeout(httpParams.getTimeout() == null ? DEFAULT_TIMEOUT * 1000 : httpParams.getTimeout() * 1000);
}
private HttpRequest buildhutoolRequest(HttpParams httpParams) {
HttpRequest request;
switch (httpParams.getMethod()) {
case PUT_REQUEST_METHOD:
request = HttpRequest.put(httpParams.url);
break;
case DELETE_REQUEST_METHOD:
request = HttpRequest.delete(httpParams.url);
break;
case POST_REQUEST_METHOD:
request = HttpRequest.post(httpParams.url);
break;
default:
request = HttpRequest.get(httpParams.url);
break;
}
if (httpParams.getHeaders() != null) {
httpParams.getHeaders().forEach(request::header);
}
if (httpParams.getBody() != null && (httpParams.getMethod().equals(POST_REQUEST_METHOD) || httpParams.getMethod().equals(PUT_REQUEST_METHOD) || httpParams.getMethod().equals(DELETE_REQUEST_METHOD))) {
request.body(httpParams.getBody(), httpParams.getMediaType());
}
request.timeout(httpParams.getTimeout());
return request;
}
@Data
public static class HttpParams {
private String method;
private String url;
private String mediaType;
private String body;
private Map<String, String> headers;
private Integer timeout;
}
// Logging methods
private void logInfo(String msg, Object... params) {
SnailJobLog.REMOTE.info("[snail-job] " + msg, params);
}
private void logWarn(String msg, Object... params) {
SnailJobLog.REMOTE.warn("[snail-job] " + msg, params);
}
}

View File

@ -0,0 +1,264 @@
package com.aizuda.snailjob.client.job.core.executor;
import com.aizuda.snailjob.client.common.config.SnailJobProperties;
import com.aizuda.snailjob.common.core.util.SnailJobFileUtil;
import com.aizuda.snailjob.common.core.util.SnailJobSystemUtil;
import com.aizuda.snailjob.client.model.ExecuteResult;
import com.aizuda.snailjob.common.core.context.SnailSpringContext;
import com.aizuda.snailjob.common.core.exception.SnailJobInnerExecutorException;
import com.aizuda.snailjob.common.log.SnailJobLog;
import com.google.common.collect.Sets;
import io.micrometer.common.util.StringUtils;
import java.io.*;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public abstract class AbstractScriptExecutor {
private static final Set<String> DOWNLOAD_PROTOCOL = Sets.newHashSet("http", "https", "ftp");
// 将所有协议组合成一个正则表达式使用 "|" 分隔表示或操作
private static final String PROTOCOLS_PATTERN = String.join("|", DOWNLOAD_PROTOCOL);
// 编译正则表达式创建 Pattern 对象作为常量使用
private static final Pattern PROTOCOL_PATTERN = Pattern.compile("^(" + PROTOCOLS_PATTERN + ")");
protected static final String SH_SHELL = "/bin/sh";
protected static final String CMD_SHELL = "cmd.exe";
private static final String READ_PATH = "READPATH:";
private static final String WORKER_DIR = SnailFileUtils.workspace() + "/script_processor/";
protected ExecuteResult process(Long taskBatchId, String scriptParams) {
logInfo("ScriptProcessor start to process, params: {}", scriptParams);
if (scriptParams == null) {
logWarn("ScriptParams is null, please check jobParam configuration.");
return ExecuteResult.failure("ScriptParams is null.");
}
String scriptPath = prepareScriptFile(taskBatchId, scriptParams);
logInfo("Generate executable file successfully, path: {}", scriptPath);
if (SnailJobSystemUtil.isOsWindows() && SH_SHELL.equals(getRunCommand())) {
logWarn("Current OS is {} where shell scripts cannot run.", SnailJobSystemUtil.getOsName());
return ExecuteResult.failure("Shell scripts cannot run on Windows.");
}
if (!SnailJobSystemUtil.isOsWindows()) {
setScriptPermissions(scriptPath);
}
return executeScript(scriptPath);
}
private String prepareScriptFile(Long taskBatchId, String processorInfo) {
String scriptPath = WORKER_DIR + getScriptName(taskBatchId);
File script = new File(scriptPath);
scriptPath = script.getAbsolutePath();
if (script.exists()) {
return scriptPath;
}
// 创建脚本目录
ensureScriptDirectory(script);
// 是否是本地目录
if (processorInfo.startsWith(READ_PATH)) {
return handleLocalScript(script, scriptPath, processorInfo);
}
// 是否为下载
// 如果是下载链接则从网络获取
Matcher matcher = PROTOCOL_PATTERN.matcher(processorInfo);
if (matcher.find()) {
try {
SnailJobFileUtil.downloadFile(processorInfo, script, 5000, 300000);
} catch (IOException e) {
throw new SnailJobInnerExecutorException("[snail-job] Script download failed", e);
}
return scriptPath;
}
// 写入脚本
try {
writeScriptContent(script, processorInfo);
} catch (IOException e) {
throw new SnailJobInnerExecutorException("[snail-job] Failed to write script", e);
}
return scriptPath;
}
private String handleLocalScript(File script, String scriptPath, String processorInfo) {
// 去掉 "READPATH:" 前缀
String newProcessorInfo = processorInfo.substring(READ_PATH.length()).trim();
File routhFile = new File(newProcessorInfo);
// 判断文件是否存在
if (routhFile.exists()) {
// 读取文件内容并写入到 script
try (BufferedReader br = new BufferedReader(new FileReader(routhFile));
BufferedWriter bw = new BufferedWriter(new FileWriter(script))) {
String line;
while ((line = br.readLine()) != null) {
bw.write(line);
bw.newLine();
}
bw.flush();
} catch (IOException e) {
throw new SnailJobInnerExecutorException("[snail-job] Local script write exception", e);
}
return scriptPath;
} else {
throw new SnailJobInnerExecutorException("File not found: {" + newProcessorInfo + "}");
}
}
private void ensureScriptDirectory(File script) {
try {
File parentDir = script.getParentFile();
if (!parentDir.exists()) {
logInfo("Script directory does not exist, creating: {}", parentDir.getAbsolutePath());
SnailJobFileUtil.mkdirs(parentDir);
}
} catch (SnailJobInnerExecutorException e) {
throw new SnailJobInnerExecutorException("[snail-job] ensure script directory error", e);
}
}
private void writeScriptContent(File script, String processorInfo) throws IOException {
try (BufferedWriter writer = Files.newBufferedWriter(script.toPath(), getCharset())) {
writer.write(processorInfo);
logInfo("Script content written successfully to: {}", script.getAbsolutePath());
}
}
private void setScriptPermissions(String scriptPath) {
ProcessBuilder chmodPb = new ProcessBuilder("/bin/chmod", "755", scriptPath);
try {
chmodPb.start().waitFor();
} catch (InterruptedException | IOException e) {
throw new SnailJobInnerExecutorException("[snail-job] Failed to set script permissions", e);
}
logInfo("chmod 755 authorization complete, ready to start execution~");
}
private ExecuteResult executeScript(String scriptPath) {
ProcessBuilder pb = getProcessBuilder(scriptPath);
Process process = null;
ExecuteResult executeResult;
try {
process = pb.start();
executeResult = captureOutput(process);
} catch (IOException | InterruptedException e) {
throw new SnailJobInnerExecutorException("[snail-job] Script execution failed", e);
} finally {
if (process.isAlive()) {
// 脚本执行失败 终止;
process.destroy();
try {
boolean exited = process.waitFor(5, TimeUnit.SECONDS); // 等待5秒
if (!exited) {
// 如果进程没有在5秒内终止则强制终止
process.destroyForcibly();
process.waitFor(); // 等待进程终止
}
logWarn("Script execution failed, starting to terminate script operation");
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
return executeResult;
}
private ProcessBuilder getProcessBuilder(String scriptPath) {
return getRunCommand().equals(CMD_SHELL) ?
new ProcessBuilder(getRunCommand(), "/c", scriptPath) :
new ProcessBuilder(getRunCommand(), scriptPath);
}
private ExecuteResult captureOutput(Process process) throws InterruptedException {
StringBuilder inputBuilder = new StringBuilder();
StringBuilder errorBuilder = new StringBuilder();
boolean success = process.waitFor() == 0;
captureStream(process.getInputStream(), inputBuilder);
captureStream(process.getErrorStream(), errorBuilder);
String result = formatResult(inputBuilder, errorBuilder);
logInfo(result);
return success ? ExecuteResult.success("Script executed successfully.") : ExecuteResult.failure("Script execution failed.");
}
private void captureStream(InputStream is, StringBuilder sb) {
try (BufferedReader br = new BufferedReader(new InputStreamReader(is, getCharset()))) {
String line;
while ((line = br.readLine()) != null) {
sb.append(line).append(System.lineSeparator());
}
} catch (Exception e) {
logWarn("Failed to capture stream.", e);
} finally {
closeQuietly(is);
}
}
private String formatResult(StringBuilder inputBuilder, StringBuilder errorBuilder) {
return String.format("[INPUT]: %s;[ERROR]: %s", inputBuilder, errorBuilder);
}
private void closeQuietly(Closeable closeable) {
try {
if (closeable != null) {
closeable.close();
}
} catch (IOException e) {
logWarn("Failed to close stream.", e);
}
}
protected abstract String getScriptName(Long instanceId);
protected abstract String getRunCommand();
protected Charset getCharset() {
return StandardCharsets.UTF_8;
}
// Logging methods
private void logInfo(String msg, Object... params) {
SnailJobLog.REMOTE.info("[snail-job] " + msg, params);
}
private void logWarn(String msg, Object... params) {
SnailJobLog.REMOTE.warn("[snail-job] " + msg, params);
}
public class SnailFileUtils {
/**
* 获取工作目录
*
* @return 允许用户通过启动配置文件自定义存储目录默认为 user.home
*/
public static String workspace() {
SnailJobProperties snailJobProperties = SnailSpringContext.getBean(SnailJobProperties.class);
String workspaceByDKey = snailJobProperties.getWorkspace();
if (StringUtils.isNotEmpty(workspaceByDKey)) {
SnailJobLog.LOCAL.info("[FileUtils] [workspace] use custom workspace: {}", workspaceByDKey);
return workspaceByDKey;
}
final String userHome = System.getProperty("user.home").concat("/snailJob/worker");
SnailJobLog.LOCAL.info("[FileUtils] [workspace] use user.home as workspace: {}", userHome);
return userHome;
}
}
}

View File

@ -0,0 +1,22 @@
package com.aizuda.snailjob.client.job.core.executor;
import java.nio.charset.Charset;
public class CMDExecutor extends AbstractScriptExecutor {
@Override
protected String getScriptName(Long taskBatchId) {
return String.format("cmd_%d.bat", taskBatchId);
}
@Override
protected String getRunCommand() {
return "cmd.exe";
}
@Override
protected Charset getCharset() {
return Charset.defaultCharset();
}
}

View File

@ -0,0 +1,22 @@
package com.aizuda.snailjob.client.job.core.executor;
import java.nio.charset.Charset;
public class PowerShellExecutor extends AbstractScriptExecutor {
@Override
protected String getScriptName(Long taskBatchId) {
return String.format("powershell_%d.ps1", taskBatchId);
}
@Override
protected String getRunCommand() {
return "powershell.exe";
}
@Override
protected Charset getCharset() {
return Charset.defaultCharset();
}
}

View File

@ -0,0 +1,15 @@
package com.aizuda.snailjob.client.job.core.executor;
public class ShellExecutor extends AbstractScriptExecutor {
@Override
protected String getScriptName(Long taskBatchId) {
return String.format("shell_%d.sh", taskBatchId);
}
@Override
protected String getRunCommand() {
return SH_SHELL;
}
}

View File

@ -0,0 +1,18 @@
package com.aizuda.snailjob.client.job.core.executor;
import com.aizuda.snailjob.client.job.core.annotation.JobExecutor;
import com.aizuda.snailjob.client.job.core.dto.JobArgs;
import com.aizuda.snailjob.client.model.ExecuteResult;
import org.springframework.stereotype.Component;
@Component
@JobExecutor(name = "snailJobCMDJobExecutor")
public class SnailJobCMDJobExecutor extends CMDExecutor {
public ExecuteResult jobExecute(JobArgs jobArgs) {
String scriptParam = (String) jobArgs.getJobParams();
return process(jobArgs.getTaskBatchId(), scriptParam);
}
}

View File

@ -0,0 +1,31 @@
package com.aizuda.snailjob.client.job.core.executor;
import com.aizuda.snailjob.client.job.core.annotation.JobExecutor;
import com.aizuda.snailjob.client.job.core.dto.JobArgs;
import com.aizuda.snailjob.client.model.ExecuteResult;
import com.aizuda.snailjob.common.core.constant.SystemConstants;
import com.aizuda.snailjob.common.core.util.JsonUtil;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.Map;
@Component
@JobExecutor(name = "snailJobHttpExecutor")
public class SnailJobHttpExecutor extends AbstractHttpExecutor {
public ExecuteResult jobExecute(JobArgs jobArgs) {
Object jobParams = jobArgs.getJobParams();
HttpParams httpParams = JsonUtil.parseObject((String) jobParams, HttpParams.class);
Map<String, String> hashMap = new HashMap<>(3);
hashMap.put(SystemConstants.SNAIL_JOB_CLIENT_GROUP, snailJobProperties.getGroup());
hashMap.put(SystemConstants.SNAIL_JOB_CLIENT_GROUP_TOKEN, snailJobProperties.getToken());
hashMap.put(SystemConstants.SNAIL_JOB_CLIENT_NAMESPACE, snailJobProperties.getNamespace());
Map<String, String> headers = (httpParams.getHeaders() == null || httpParams.getHeaders().isEmpty()) ? new HashMap<>() : httpParams.getHeaders();
headers.putAll(hashMap);
httpParams.setHeaders(headers);
return process(httpParams);
}
}

View File

@ -0,0 +1,18 @@
package com.aizuda.snailjob.client.job.core.executor;
import com.aizuda.snailjob.client.job.core.annotation.JobExecutor;
import com.aizuda.snailjob.client.job.core.dto.JobArgs;
import com.aizuda.snailjob.client.model.ExecuteResult;
import org.springframework.stereotype.Component;
@Component
@JobExecutor(name = "snailJobPowerShellJobExecutor")
public class SnailJobPowerShellJobExecutor extends PowerShellExecutor {
public ExecuteResult jobExecute(JobArgs jobArgs) {
String scriptParam = (String) jobArgs.getJobParams();
return process(jobArgs.getTaskBatchId(), scriptParam);
}
}

View File

@ -0,0 +1,18 @@
package com.aizuda.snailjob.client.job.core.executor;
import com.aizuda.snailjob.client.job.core.annotation.JobExecutor;
import com.aizuda.snailjob.client.job.core.dto.JobArgs;
import com.aizuda.snailjob.client.model.ExecuteResult;
import org.springframework.stereotype.Component;
@Component
@JobExecutor(name = "snailJobShellJobExecutor")
public class SnailJobShellJobExecutor extends ShellExecutor {
public ExecuteResult jobExecute(JobArgs jobArgs) {
String scriptParam = (String) jobArgs.getJobParams();
return process(jobArgs.getTaskBatchId(), scriptParam);
}
}

View File

@ -24,6 +24,21 @@ public interface SystemConstants {
*/
String SNAIL_JOB_STATUS_CODE = "519";
/**
* 客户端组名
*/
String SNAIL_JOB_CLIENT_GROUP = "group";
/**
* 客户端组对应token
*/
String SNAIL_JOB_CLIENT_GROUP_TOKEN = "token";
/**
* 命名空间
*/
String SNAIL_JOB_CLIENT_NAMESPACE = "namespace";
/**
* 默认的调用链超时时间 单位毫秒(ms)
*/

View File

@ -0,0 +1,31 @@
package com.aizuda.snailjob.common.core.exception;
public class SnailJobInnerExecutorException extends BaseSnailJobException{
public SnailJobInnerExecutorException(String message) {
super(message);
}
public SnailJobInnerExecutorException(String message, Throwable cause) {
super(message, cause);
}
public SnailJobInnerExecutorException(Throwable cause) {
super(cause);
}
public SnailJobInnerExecutorException(String message, Object... arguments) {
super(message, arguments);
}
public SnailJobInnerExecutorException(String message, Object[] arguments, Throwable cause) {
super(message, arguments, cause);
}
public SnailJobInnerExecutorException(String message, Object argument, Throwable cause) {
super(message, argument, cause);
}
public SnailJobInnerExecutorException(String message, Object argument) {
super(message, argument);
}
}

View File

@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
@ -134,6 +135,38 @@ public class JsonUtil {
return JsonMapper.toJson(jsonBytes);
}
/**
* 验证 JSON 字符串是否有效
*
* @param jsonString JSON 字符串
* @return 如果 JSON 字符串有效则返回 true否则返回 false
*/
public static boolean isValidJson(String jsonString) {
try {
JsonMapper.objectMapper.readTree(jsonString);
return true;
} catch (Exception e) {
return false;
}
}
/**
* 创建一个空的json
*
* @return String
*/
public static String toJSONString() {
// 创建一个空的 ObjectNode
ObjectNode objectNode = JsonMapper.objectMapper.createObjectNode();
try {
// ObjectNode 序列化为 JSON 字符串
return JsonMapper.objectMapper.writeValueAsString(objectNode);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* 内部类处理Json
*/

View File

@ -0,0 +1,47 @@
package com.aizuda.snailjob.common.core.util;
import com.aizuda.snailjob.common.core.exception.SnailJobInnerExecutorException;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
public class SnailJobFileUtil {
/**
* 从给定的 URL 下载文件到本地文件系统
*
* @param urlString 要下载的文件的 URL 字符串
* @param destinationFile 下载后的本地文件
* @param connectionTimeout 连接超时时间毫秒
* @param readTimeout 读取超时时间毫秒
* @throws IOException 如果发生 IO 错误
*/
public static void downloadFile(String urlString, File destinationFile, int connectionTimeout, int readTimeout) throws IOException {
URL url = new URL(urlString);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(connectionTimeout);
connection.setReadTimeout(readTimeout);
try (InputStream inputStream = new BufferedInputStream(connection.getInputStream());
FileOutputStream fileOS = new FileOutputStream(destinationFile);
BufferedOutputStream bufferedOutStream = new BufferedOutputStream(fileOS)) {
byte[] dataBuffer = new byte[1024];
int bytesRead;
while ((bytesRead = inputStream.read(dataBuffer, 0, 1024)) != -1) {
bufferedOutStream.write(dataBuffer, 0, bytesRead);
}
} finally {
connection.disconnect();
}
}
public static File mkdirs(File directory) throws SnailJobInnerExecutorException {
if (directory != null && !directory.mkdirs() && !directory.isDirectory()) {
throw new SnailJobInnerExecutorException("Cannot create directory '" + directory + "'.");
} else {
return directory;
}
}
}

View File

@ -0,0 +1,13 @@
package com.aizuda.snailjob.common.core.util;
public class SnailJobSystemUtil {
public static String getOsName(){
return System.getProperty("os.name");
}
public static boolean isOsWindows(){
String osName = getOsName();
return osName != null && osName.toLowerCase().contains("windows");
}
}