gtsoft/gtsoft-snail-job-server
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects 1 Releases Wiki Activity

feat:(unify): 新增容器和镜像管理

Browse Source
This commit is contained in:
opensnail 2024-12-15 23:10:24 +08:00
parent cbee3797fe
commit fad4b0b8c4
13 changed files with 317 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/controller/DockerController.java
View File

@ -1,6 +1,12 @@
package com.aizuda.snailjob.server.web.controller;
import com.aizuda.snailjob.server.web.model.response.ContainerVO;
import com.aizuda.snailjob.server.web.model.base.PageResult;
import com.aizuda.snailjob.server.web.model.request.BuildImageRequestVO;
import com.aizuda.snailjob.server.web.model.request.ContainerQueryVO;
import com.aizuda.snailjob.server.web.model.request.CreateContainerRequestVO;
import com.aizuda.snailjob.server.web.model.request.ImageQueryVO;
import com.aizuda.snailjob.server.web.model.response.ContainerResponseVO;
import com.aizuda.snailjob.server.web.model.response.ImageResponseVO;
import com.aizuda.snailjob.server.web.service.DockerService;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.InspectContainerResponse;
@ -14,12 +20,11 @@ import java.util.List;
@RequestMapping("/docker")
@RequiredArgsConstructor
public class DockerController {
private final DockerService dockerService;
@GetMapping("/container/list")
public List<ContainerVO> getContainerList(@RequestParam("containerName") String containerName) {
return dockerService.getContainerList(containerName);
@GetMapping("/container/page/list")
public PageResult<List<ContainerResponseVO>> getContainerList(ContainerQueryVO containerQueryVO) {
return dockerService.getContainerList(containerQueryVO);
}
@GetMapping("/container/{id}")
@ -32,13 +37,23 @@ public class DockerController {
return dockerService.getContainerLogById(id);
}
@GetMapping("/create/container")
public boolean createContainer(@RequestParam("containerName") String containerName, @RequestParam("imageName") String imageName) {
return dockerService.createContainer(containerName, imageName);
@PostMapping("/create/container")
public boolean createContainer(@RequestBody CreateContainerRequestVO requestVO) {
return dockerService.createContainer(requestVO);
}
@GetMapping("/build/image")
public boolean buildImage(@RequestParam("imagerName") String imagerName, @RequestParam("path") String path) {
return dockerService.buildImage(imagerName, path);
@GetMapping("/image/page/list")
public PageResult<List<ImageResponseVO>> getImageList(ImageQueryVO imageQueryVO) {
return dockerService.getImageList(imageQueryVO);
}
@PostMapping("/build/image")
public String buildImage(@RequestBody BuildImageRequestVO requestVO) {
return dockerService.buildImage(requestVO);
}
@PostMapping("/quick/publish")
public boolean quickPublish(@RequestBody CreateContainerRequestVO requestVO) {
return dockerService.quickPublish(requestVO);
}
}

11
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/controller/PythonController.java
View File

@ -10,8 +10,19 @@ import org.springframework.web.bind.annotation.*;
@RequiredArgsConstructor
public class PythonController {
private final PythonService pythonService;
@PostMapping("/run")
public Boolean run(@RequestBody RunPythonRequestVO requestVO) {
return pythonService.runPython(requestVO);
}
@PostMapping("/stop")
public Boolean stop() {
return pythonService.stopPython();
}
@GetMapping("/status")
public Boolean getStatus() {
return pythonService.getStatus();
}
}

5
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/base/SjBuildImageResultCallback.java
View File

@ -1,5 +1,6 @@
package com.aizuda.snailjob.server.web.model.base;
import cn.hutool.core.util.StrUtil;
import com.github.dockerjava.api.async.ResultCallbackTemplate;
import com.github.dockerjava.api.exception.DockerClientException;
import com.github.dockerjava.api.model.BuildResponseItem;
@ -23,6 +24,10 @@ public class SjBuildImageResultCallback extends ResultCallbackTemplate<com.githu
} else if (item.isErrorIndicated()) {
this.error = item.getError();
}
String stream = item.getStream();
if (StrUtil.isBlank(stream)) {
return;
}
LOGGER.info("{}", item.getStream());
}

11
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/request/BuildImageRequestVO.java Normal file
View File

@ -0,0 +1,11 @@
package com.aizuda.snailjob.server.web.model.request;
import lombok.Data;
@Data
public class BuildImageRequestVO {
private String groupName;
private String namespaceId;
}

12
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/request/ContainerQueryVO.java Normal file
View File

@ -0,0 +1,12 @@
package com.aizuda.snailjob.server.web.model.request;
import com.aizuda.snailjob.server.web.model.base.BaseQueryVO;
import lombok.Data;
import lombok.EqualsAndHashCode;
@EqualsAndHashCode(callSuper = true)
@Data
public class ContainerQueryVO extends BaseQueryVO {
private String containerName;
private String groupName;
}

13
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/request/CreateContainerRequestVO.java Normal file
View File

@ -0,0 +1,13 @@
package com.aizuda.snailjob.server.web.model.request;
import lombok.Data;
@Data
public class CreateContainerRequestVO {
private String groupName;
private String namespaceId;
private String imageName;
}

10
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/request/ImageQueryVO.java Normal file
View File

@ -0,0 +1,10 @@
package com.aizuda.snailjob.server.web.model.request;
import com.aizuda.snailjob.server.web.model.base.BaseQueryVO;
import lombok.Data;
import lombok.EqualsAndHashCode;
@EqualsAndHashCode(callSuper = true)
@Data
public class ImageQueryVO extends BaseQueryVO {
}

3
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/response/ContainerVO.java → snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/response/ContainerResponseVO.java
View File

@ -1,12 +1,11 @@
package com.aizuda.snailjob.server.web.model.response;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import java.util.List;
@Data
public class ContainerVO {
public class ContainerResponseVO {
private String id;

18
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/model/response/ImageResponseVO.java Normal file
View File

@ -0,0 +1,18 @@
package com.aizuda.snailjob.server.web.model.response;
import lombok.Data;
import java.util.Map;
@Data
public class ImageResponseVO {
private Long created;
private String id;
private String groupName;
private String[] repoTags;
private Long size;
public Map<String, String> labels;
private boolean isUsed;
}

18
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/DockerService.java
View File

@ -1,6 +1,12 @@
package com.aizuda.snailjob.server.web.service;
import com.aizuda.snailjob.server.web.model.response.ContainerVO;
import com.aizuda.snailjob.server.web.model.base.PageResult;
import com.aizuda.snailjob.server.web.model.request.BuildImageRequestVO;
import com.aizuda.snailjob.server.web.model.request.ContainerQueryVO;
import com.aizuda.snailjob.server.web.model.request.CreateContainerRequestVO;
import com.aizuda.snailjob.server.web.model.request.ImageQueryVO;
import com.aizuda.snailjob.server.web.model.response.ContainerResponseVO;
import com.aizuda.snailjob.server.web.model.response.ImageResponseVO;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.model.Frame;
@ -9,13 +15,17 @@ import java.util.List;
public interface DockerService {
boolean buildImage(String groupName, String path);
String buildImage(BuildImageRequestVO requestVO);
boolean createContainer(String containerName, String imageName);
boolean createContainer(CreateContainerRequestVO requestVO);
List<ContainerVO> getContainerList(String containerName);
PageResult<List<ContainerResponseVO>> getContainerList(ContainerQueryVO containerQueryVO);
InspectContainerResponse getContainerById(String id);
ResultCallback<Frame> getContainerLogById(String id) throws InterruptedException;
boolean quickPublish(CreateContainerRequestVO requestVO);
PageResult<List<ImageResponseVO>> getImageList(ImageQueryVO imageQueryVO);
}

4
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/PythonService.java
View File

@ -6,4 +6,8 @@ public interface PythonService {
boolean runPython(RunPythonRequestVO pythonPath);
boolean stopPython();
Boolean getStatus();
}

136
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/impl/DockerServiceImpl.java
View File

@ -1,9 +1,18 @@
package com.aizuda.snailjob.server.web.service.impl;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.lang.Assert;
import com.aizuda.snailjob.common.core.util.StreamUtils;
import com.aizuda.snailjob.server.common.exception.SnailJobServerException;
import com.aizuda.snailjob.server.common.util.DateUtils;
import com.aizuda.snailjob.server.web.model.base.PageResult;
import com.aizuda.snailjob.server.web.model.base.SjBuildImageResultCallback;
import com.aizuda.snailjob.server.web.model.response.ContainerVO;
import com.aizuda.snailjob.server.web.model.request.BuildImageRequestVO;
import com.aizuda.snailjob.server.web.model.request.ContainerQueryVO;
import com.aizuda.snailjob.server.web.model.request.CreateContainerRequestVO;
import com.aizuda.snailjob.server.web.model.request.ImageQueryVO;
import com.aizuda.snailjob.server.web.model.response.ContainerResponseVO;
import com.aizuda.snailjob.server.web.model.response.ImageResponseVO;
import com.aizuda.snailjob.server.web.service.DockerService;
import com.aizuda.snailjob.server.web.service.handler.DockerHandler;
import com.github.dockerjava.api.DockerClient;
@ -13,6 +22,7 @@ import com.github.dockerjava.api.command.*;
import com.github.dockerjava.api.model.Container;
import com.github.dockerjava.api.model.ContainerPort;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.Image;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import lombok.RequiredArgsConstructor;
@ -30,40 +40,44 @@ import static com.aizuda.snailjob.server.common.util.DateUtils.PURE_DATETIME_MS_
@RequiredArgsConstructor
public class DockerServiceImpl implements DockerService {
private final DockerHandler dockerHandler;
private static final String workdir = "/Users/zhangshuguang/snail-job-python";
@Override
public boolean buildImage(String groupName, String path) {
String format = String.format("sj_%s_%s", groupName, DateUtils.toNowFormat(PURE_DATETIME_MS_PATTERN));
public String buildImage(BuildImageRequestVO requestVO) {
String imageName = String.format("sj_%s_%s", requestVO.getGroupName(), DateUtils.toNowFormat(PURE_DATETIME_MS_PATTERN));
List<File> dockerfileList = FileUtil.loopFiles(workdir, pathname -> pathname.getName().equals("Dockerfile"));
Assert.isFalse(dockerfileList.isEmpty(), () -> new SnailJobServerException("不存在Dockerfile文件"));
Assert.isTrue(dockerfileList.size() == 1, () -> new SnailJobServerException("存在多个Dockerfile文件"));
File file = dockerfileList.get(0);
buildImage(file, imageName);
return imageName;
}
private void buildImage(File file, String imageName) {
DockerClient dockerClient = dockerHandler.getDockerClient();
BuildImageCmd imageCmd = dockerClient
.buildImageCmd(new File(path))
.withTags(Sets.newHashSet(format));
.buildImageCmd(file)
.withTags(Sets.newHashSet(imageName));
imageCmd.exec(new SjBuildImageResultCallback()).awaitImageId();
return false;
}
@Override
public boolean createContainer(String containerName, String imageName) {
public boolean createContainer(CreateContainerRequestVO requestVO) {
DockerClient dockerClient = dockerHandler.getDockerClient();
String image = String.format("%s:latest", imageName);
containerName = String.format("sj_%s_%s", containerName, DateUtils.toNowFormat(PURE_DATETIME_MS_PATTERN));
String image = String.format("%s:latest", requestVO.getImageName());
String containerName = String.format("sj-%s", requestVO.getGroupName());
// Pull an image
// try {
// dockerClient.pullImageCmd(String.format("%s:latest", imageName)).start().awaitCompletion();
// } catch (InterruptedException e) {
// throw new RuntimeException(e);
// }
Map<String, String> labels = Maps.newHashMap();
labels.put("name", "snail_job");
labels.put("groupName", requestVO.getGroupName());
labels.put("namespaceId", requestVO.getNamespaceId());
// Create a container
CreateContainerResponse container = dockerClient
.createContainerCmd(image)
.withName(containerName)
.withLabels(labels)
// .withCmd("echo", "Hello, Docker-Java!")
.exec();
// Start the container
@ -73,26 +87,29 @@ public class DockerServiceImpl implements DockerService {
}
@Override
public List<ContainerVO> getContainerList(String containerName) {
public PageResult<List<ContainerResponseVO>> getContainerList(ContainerQueryVO containerQueryVO) {
DockerClient dockerClient = dockerHandler.getDockerClient();
Map<String, String> labels = Maps.newHashMap();
labels.put("app-name", "snail-job-server");
labels.put("app-name", "snail-job-python-client");
ListContainersCmd listContainersCmd = dockerClient.listContainersCmd();
List<Container> containerList = listContainersCmd.withLabelFilter(labels).withShowAll(true).exec();
List<Container> containerList = listContainersCmd
// .withNameFilter()
.withLabelFilter(labels)
.withShowAll(true).exec();
return StreamUtils.toList(containerList, container -> {
ContainerVO containerVO = new ContainerVO();
containerVO.setId(container.getId());
containerVO.setImage(container.getImage());
containerVO.setName(container.getNames()[0]);
containerVO.setStatus(container.getStatus());
containerVO.setState(container.getState());
List<ContainerVO.ContainerPortVO> portVOS = StreamUtils.toList(Arrays.stream(container.getPorts()).toList(), new Function<ContainerPort, ContainerVO.ContainerPortVO>() {
List<ContainerResponseVO> list = StreamUtils.toList(containerList, container -> {
ContainerResponseVO containerResponseVO = new ContainerResponseVO();
containerResponseVO.setId(container.getId());
containerResponseVO.setImage(container.getImage());
containerResponseVO.setName(container.getNames()[0]);
containerResponseVO.setStatus(container.getStatus());
containerResponseVO.setState(container.getState());
List<ContainerResponseVO.ContainerPortVO> portVOS = StreamUtils.toList(Arrays.stream(container.getPorts()).toList(), new Function<ContainerPort, ContainerResponseVO.ContainerPortVO>() {
@Override
public ContainerVO.ContainerPortVO apply(ContainerPort containerPort) {
ContainerVO.ContainerPortVO portVO = new ContainerVO.ContainerPortVO();
public ContainerResponseVO.ContainerPortVO apply(ContainerPort containerPort) {
ContainerResponseVO.ContainerPortVO portVO = new ContainerResponseVO.ContainerPortVO();
portVO.setPublicPort(containerPort.getPublicPort());
portVO.setPrivatePort(containerPort.getPrivatePort());
portVO.setIp(containerPort.getIp());
@ -101,9 +118,17 @@ public class DockerServiceImpl implements DockerService {
}
});
containerVO.setPorts(portVOS);
return containerVO;
containerResponseVO.setPorts(portVOS);
return containerResponseVO;
});
PageResult<List<ContainerResponseVO>> pageResult = new PageResult<>();
pageResult.setPage(1);
pageResult.setSize(10);
pageResult.setTotal(list.size());
pageResult.setData(list);
return pageResult;
}
@Override
@ -128,4 +153,49 @@ public class DockerServiceImpl implements DockerService {
}
}).awaitStarted();
}
@Override
public boolean quickPublish(CreateContainerRequestVO requestVO) {
BuildImageRequestVO buildImageRequestVO = new BuildImageRequestVO();
buildImageRequestVO.setGroupName(requestVO.getGroupName());
buildImageRequestVO.setNamespaceId(requestVO.getNamespaceId());
requestVO.setImageName(buildImage(buildImageRequestVO));
createContainer(requestVO);
return false;
}
@Override
public PageResult<List<ImageResponseVO>> getImageList(ImageQueryVO imageQueryVO) {
DockerClient dockerClient = dockerHandler.getDockerClient();
ListImagesCmd listImagesCmd = dockerClient.listImagesCmd();
List<Image> images = listImagesCmd
// todo 过滤
// .withLabelFilter()
.withShowAll(false)
.exec();
List<Container> containerList = dockerClient.listContainersCmd().withShowAll(true).exec();
List<ImageResponseVO> imageResponseList = images.stream().map(image -> {
ImageResponseVO imageResponseVO = new ImageResponseVO();
imageResponseVO.setId(image.getId());
imageResponseVO.setRepoTags(image.getRepoTags());
imageResponseVO.setCreated(image.getCreated());
Map<String, String> labels = image.getLabels();
if (labels != null) {
imageResponseVO.setGroupName(labels.get("group-name"));
}
imageResponseVO.setUsed(containerList.stream().anyMatch(container -> container.getId().equals(image.getId())));
return imageResponseVO;
}).toList();
PageResult<List<ImageResponseVO>> pageResult = new PageResult<>();
pageResult.setPage(1);
pageResult.setSize(10);
pageResult.setTotal(images.size());
pageResult.setData(imageResponseList);
return pageResult;
}
}

96
snail-job-server/snail-job-server-web/src/main/java/com/aizuda/snailjob/server/web/service/impl/PythonServiceImpl.java
View File

@ -1,22 +1,92 @@
package com.aizuda.snailjob.server.web.service.impl;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.StrUtil;
import com.aizuda.snailjob.common.core.util.StreamUtils;
import com.aizuda.snailjob.server.common.exception.SnailJobServerException;
import com.aizuda.snailjob.server.web.model.request.RunPythonRequestVO;
import com.aizuda.snailjob.server.web.service.PythonService;
import com.google.common.collect.Lists;
import org.springframework.stereotype.Service;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@Service
public class PythonServiceImpl implements PythonService {
private static final String MAIN_PY = "main.py";
private static final String REQUIREMENTS_TXT = "requirements.txt";
private static final String workdir = "/Users/zhangshuguang/";
private static final String workdir = "/Users/zhangshuguang/snail-job-python";
@Override
public boolean runPython(RunPythonRequestVO requestVO) {
return execPython(requestVO);
new Thread(() -> execPython(requestVO)).start();
return true;
}
@Override
public boolean stopPython() {
// todo 端口看怎么获取
killProcessOnPort(17889);
return false;
}
@Override
public Boolean getStatus() {
try {
// 构造 lsof 命令
ProcessBuilder lsofBuilder = new ProcessBuilder("lsof", "-t", "-i:" + 17889);
Process lsofProcess = lsofBuilder.start();
// 获取 lsof 输出 (进程 ID)
BufferedReader reader = new BufferedReader(new InputStreamReader(lsofProcess.getInputStream()));
String pid = reader.readLine(); // 获取第一个进程 ID
if (StrUtil.isNotBlank(pid)) {
return true;
}
} catch (Exception e) {
e.printStackTrace();
}
return false;
}
public void killProcessOnPort(int port) {
try {
// 构造 lsof 命令
ProcessBuilder lsofBuilder = new ProcessBuilder("lsof", "-t", "-i:" + port);
Process lsofProcess = lsofBuilder.start();
// 获取 lsof 输出 (进程 ID)
BufferedReader reader = new BufferedReader(new InputStreamReader(lsofProcess.getInputStream()));
String pid = reader.readLine(); // 获取第一个进程 ID
if (pid != null) {
System.out.println("Found process with PID: " + pid + " on port: " + port);
// 构造 kill 命令
ProcessBuilder killBuilder = new ProcessBuilder("kill", "-9", pid);
Process killProcess = killBuilder.start();
int killExitCode = killProcess.waitFor(); // 等待命令执行完成
if (killExitCode == 0) {
System.out.println("Successfully killed process with PID: " + pid);
} else {
System.err.println("Failed to kill process with PID: " + pid);
}
} else {
System.out.println("No process found running on port: " + port);
}
} catch (IOException | InterruptedException e) {
e.printStackTrace();
}
}
public boolean execPython(RunPythonRequestVO requestVO) {
@ -25,12 +95,23 @@ public class PythonServiceImpl implements PythonService {
String command = requestVO.getCommand();
String[] split = command.split("&&");
// Python 脚本路径
String pythonScriptPath = workdir + StrUtil.SLASH + requestVO.getPythonPath();
String pythonScriptTxt = workdir + StrUtil.SLASH + StrUtil.replaceFirst(requestVO.getPythonPath(), "main.py", "requirements.txt");
List<File> dockerfileList = FileUtil.loopFiles(workdir, pathname -> pathname.getName().equals(MAIN_PY) || pathname.getName().endsWith(REQUIREMENTS_TXT));
List<File> mainFiles = StreamUtils.filter(dockerfileList, file -> file.getName().equals(MAIN_PY));
Assert.isFalse(mainFiles.isEmpty(), () -> new SnailJobServerException("不存在{}文件", MAIN_PY));
Assert.isTrue(mainFiles.size() == 1, () -> new SnailJobServerException("存在多个{}}文件", MAIN_PY));
List<File> requirementsFiles = StreamUtils.filter(dockerfileList, file -> file.getName().equals(REQUIREMENTS_TXT));
Assert.isFalse(requirementsFiles.isEmpty(), () -> new SnailJobServerException("不存在{}文件", REQUIREMENTS_TXT));
Assert.isTrue(requirementsFiles.size() == 1, () -> new SnailJobServerException("存在多个{}}文件", REQUIREMENTS_TXT));
// 构建命令
ProcessBuilder pipBuilder = new ProcessBuilder(split[0], pythonScriptTxt);
List<String> list = Arrays.stream(split[0].split(" ")).map(String::trim).collect(Collectors.toList());
list.add(requirementsFiles.get(0).getPath());
ProcessBuilder pipBuilder = new ProcessBuilder(list);
pipBuilder.redirectErrorStream(true);
// 启动进程
@ -45,7 +126,7 @@ public class PythonServiceImpl implements PythonService {
return false;
}
// 构建命令
ProcessBuilder processBuilder = new ProcessBuilder(split[1], pythonScriptPath);
ProcessBuilder processBuilder = new ProcessBuilder(Lists.newArrayList(split[1].trim(), mainFiles.get(0).getPath()));
processBuilder.redirectErrorStream(true);
// 启动进程
@ -71,6 +152,7 @@ public class PythonServiceImpl implements PythonService {
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
// todo 主动推送日志到页面
System.out.println(line);
}
}