Skip to content

增加了大量爬取的功能，修改了爬取逻辑使得它能够直接爬取最新漏洞数据 #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

增加了大量爬取的功能，修改了爬取逻辑使得它能够直接爬取最新漏洞数据 #5

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
252 changes: 142 additions & 110 deletions src/main/java/xin/ctkqiang/controller/ExploitDbController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,17 +85,12 @@ private int create(String id, String description, String date, String author, St
* @param extension 导出文件的扩展名（支持：csv, json, sql）
*/
public void Crawl(int pageSize, boolean isExport, String extension) {
// 构建API请求URL，包含分页参数
String url = String.format(ExploitDbController.URL + "search?draw=1&start=%d&length=%d", 0, pageSize);

// 初始化状态码和结果列表
int Status = 0;
List<Exploit> exploits = new ArrayList<>();

// 获取数据总条数
int total = 0;
try {
// 构建HTTP请求，设置必要的请求头
HttpRequest Request = HttpRequest.newBuilder()
.uri(URI.create(url))
String probeUrl = ExploitDbController.URL + "search?draw=1&start=0&length=1";
HttpRequest probeReq = HttpRequest.newBuilder()
.uri(URI.create(probeUrl))
.header("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
.header("Accept", "application/json, text/javascript, */*; q=0.01")
Expand All @@ -104,127 +99,164 @@ public void Crawl(int pageSize, boolean isExport, String extension) {
.header("X-Requested-With", "XMLHttpRequest")
.GET()
.build();
HttpResponse<String> probeResp = Client.send(probeReq, HttpResponse.BodyHandlers.ofString());
total = new ObjectMapper().readTree(probeResp.body()).get("recordsTotal").asInt();
System.out.println("tatal: " + total);
} catch (Exception e) {
e.printStackTrace();
}
// 构建API请求URL，包含分页参数
int want = Math.min(pageSize, total);
// String url = String.format(ExploitDbController.URL + "search?draw=1&start=%d&length=%d", start, pageSize);

// 发送HTTP请求并获取响应
HttpResponse<String> Response = Client.send(Request, HttpResponse.BodyHandlers.ofString());

// 调试模式下打印响应预览
if (debug) {
System.out.println("响应内容预览：" + Response.body().substring(0, 500));
}

// 解析JSON响应
JsonNode Root = this.Mapper.readTree(Response.body());
JsonNode Data = Root.get("data");

// 处理数据数组
if (Data.isArray()) {
// 确保数据有效性
assert Data.isArray() && Data.size() > 0 : "Data 不是一个数组或大小为0";

// 输出进度信息
System.out.println("正在爬取 Exploit-DB...");
System.out.println("共找到 " + Data.size() + " 条记录。");
System.out.println("正在写入数据库...");
// 初始化状态码和结果列表
int Status = 0;
List<Exploit> exploits = new ArrayList<>();

// 调试模式下打印完整数据
final int PAGE = 100;
for (int end = total; end > 0 && exploits.size() < want; end -= PAGE) {
int start = Math.max(Math.max(end - PAGE, 0), end - want);
int len = Math.min(PAGE, want - exploits.size());
String url = String.format(URL + "search?draw=1&start=%d&length=%d", start, len);
try {
// 构建HTTP请求，设置必要的请求头
HttpRequest Request = HttpRequest.newBuilder()
.uri(URI.create(url))
.header("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
.header("Accept", "application/json, text/javascript, */*; q=0.01")
.header("Accept-Language", "en-US,en;q=0.9")
.header("Referer", ExploitDbController.URL)
.header("X-Requested-With", "XMLHttpRequest")
.GET()
.build();

// 发送HTTP请求并获取响应
HttpResponse<String> Response = Client.send(Request, HttpResponse.BodyHandlers.ofString());

// 调试模式下打印响应预览
if (debug) {
System.out.println(Data);
System.out.println("响应内容预览：" + Response.body().substring(0, 500));
}

// 遍历每条漏洞记录并解析
for (JsonNode Node : Data) {
Exploit exploit = new Exploit();
// 解析JSON响应
JsonNode Root = this.Mapper.readTree(Response.body());
JsonNode Data = Root.get("data");

// 设置漏洞ID
exploit.setId(Node.get("id").asText());
// 处理数据数组
if (Data.isArray()) {
// 确保数据有效性
assert Data.isArray() && Data.size() > 0 : "Data 不是一个数组或大小为0";

// 获取各字段节点
JsonNode descriptionNode = Node.get("description");
JsonNode authorNode = Node.get("author");
JsonNode typeNode = Node.get("type");
JsonNode platformNode = Node.get("platform");
// 输出进度信息
System.out.println("正在爬取 Exploit-DB...");
System.out.println("共找到 " + Data.size() + " 条记录。");
System.out.println("正在写入数据库...");

// 解析描述信息（数组格式）
if (descriptionNode != null && descriptionNode.isArray() && descriptionNode.size() > 1) {
exploit.setDescription(descriptionNode.get(1).asText());
} else {
exploit.setDescription("");
// 调试模式下打印完整数据
if (debug) {
System.out.println(Data);
}

// 解析发布日期
if (Node.has("date_published")) {
exploit.setDate(Node.get("date_published").asText());
} else {
exploit.setDate("");
}
// 遍历每条漏洞记录并解析
for (JsonNode Node : Data) {
Exploit exploit = new Exploit();

// 设置漏洞ID
exploit.setId(Node.get("id").asText());

// 获取各字段节点
JsonNode descriptionNode = Node.get("description");
JsonNode authorNode = Node.get("author");
JsonNode typeNode = Node.get("type");
JsonNode platformNode = Node.get("platform");
// 添加获取CVE编号的节点
JsonNode codeNode = Node.get("code");

// 解析描述信息（数组格式）
if (descriptionNode != null && descriptionNode.isArray() && descriptionNode.size() > 1) {
exploit.setDescription(descriptionNode.get(1).asText());
} else {
exploit.setDescription("");
}

// 解析作者信息
if (authorNode != null && authorNode.has("name")) {
exploit.setAuthor(authorNode.get("name").asText());
} else {
exploit.setAuthor("");
}
// 解析发布日期
if (Node.has("date_published")) {
exploit.setDate(Node.get("date_published").asText());
} else {
exploit.setDate("");
}

// 解析漏洞类型
if (typeNode != null && typeNode.has("name")) {
exploit.setType(typeNode.get("name").asText());
} else {
exploit.setType("");
}
// 解析作者信息
if (authorNode != null && authorNode.has("name")) {
exploit.setAuthor(authorNode.get("name").asText());
} else {
exploit.setAuthor("");
}

// 解析影响平台
if (platformNode != null && platformNode.has("platform")) {
exploit.setPlatform(platformNode.get("platform").asText());
} else {
exploit.setPlatform("");
}
// 解析漏洞类型
if (typeNode != null && typeNode.has("name")) {
exploit.setType(typeNode.get("name").asText());
} else {
exploit.setType("");
}

// 添加到结果列表
exploits.add(exploit);
}
// 将爬取的数据保存到数据库
Status = super.AddExploit(exploits);

// 如果数据保存成功
if (Status != (-1)) {
// 在控制台打印数据表格
ExploitDbController.PrintExploitTable(exploits);

// 如果需要导出数据
if (isExport) {
int status;

// 根据指定的文件格式导出数据
switch (extension) {
case "sql":
status = super.ExportToSQL(exploits);
break;
case "json":
status = super.ExportToJSON(exploits);
break;
case "yaml":
status = super.ExportToYAML(exploits);
break;
case "csv":
default:
status = super.ExportToCSV(exploits);
break;
// 解析影响平台
if (platformNode != null && platformNode.has("platform")) {
exploit.setPlatform(platformNode.get("platform").asText());
} else {
exploit.setPlatform("");
}

// 显示导出状态
this.ExportStatus(status, extension);
// 添加到结果列表
exploits.add(exploit);
if (exploits.size() >= want)
break;
}
}
Thread.sleep(1000);
} catch (IOException | InterruptedException e) {
String msg = e.getMessage();

if (msg != null && BLOCK_KEYWORDS.stream().anyMatch(k -> msg.toLowerCase().contains(k))) {
System.err.println("爬取 Exploit-DB : ⚠️ 请求被阻止，可能遭遇防火墙、访问限制或代理问题，请检查网络配置。");
} else {
System.err.println("爬取 Exploit-DB 时出错：" + msg);
}
}
}

} catch (IOException | InterruptedException e) {
String msg = e.getMessage();
// 将爬取的数据保存到数据库
Status = super.AddExploit(exploits);

// 如果数据保存成功
if (Status != (-1)) {
// 在控制台打印数据表格
ExploitDbController.PrintExploitTable(exploits);

// 如果需要导出数据
if (isExport) {
int status;

// 根据指定的文件格式导出数据
switch (extension) {
case "sql":
status = super.ExportToSQL(exploits);
break;
case "json":
status = super.ExportToJSON(exploits);
break;
case "yaml":
status = super.ExportToYAML(exploits);
break;
case "csv":
default:
status = super.ExportToCSV(exploits);
break;
}

if (msg != null && BLOCK_KEYWORDS.stream().anyMatch(k -> msg.toLowerCase().contains(k))) {
System.err.println("爬取 Exploit-DB : ⚠️ 请求被阻止，可能遭遇防火墙、访问限制或代理问题，请检查网络配置。");
} else {
System.err.println("爬取 Exploit-DB 时出错：" + msg);
// 显示导出状态
this.ExportStatus(status, extension);
}
}

Expand Down