diff --git a/src/main/java/xin/ctkqiang/controller/ExploitDbController.java b/src/main/java/xin/ctkqiang/controller/ExploitDbController.java index 98bacad..3553af0 100644 --- a/src/main/java/xin/ctkqiang/controller/ExploitDbController.java +++ b/src/main/java/xin/ctkqiang/controller/ExploitDbController.java @@ -85,17 +85,12 @@ private int create(String id, String description, String date, String author, St * @param extension 导出文件的扩展名（支持：csv, json, sql） */ public void Crawl(int pageSize, boolean isExport, String extension) { - // 构建API请求URL，包含分页参数 - String url = String.format(ExploitDbController.URL + "search?draw=1&start=%d&length=%d", 0, pageSize); - - // 初始化状态码和结果列表 - int Status = 0; - List exploits = new ArrayList<>(); - + // 获取数据总条数 + int total = 0; try { - // 构建HTTP请求，设置必要的请求头 - HttpRequest Request = HttpRequest.newBuilder() - .uri(URI.create(url)) + String probeUrl = ExploitDbController.URL + "search?draw=1&start=0&length=1"; + HttpRequest probeReq = HttpRequest.newBuilder() + .uri(URI.create(probeUrl)) .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36") .header("Accept", "application/json, text/javascript, */*; q=0.01") @@ -104,127 +99,164 @@ public void Crawl(int pageSize, boolean isExport, String extension) { .header("X-Requested-With", "XMLHttpRequest") .GET() .build(); + HttpResponse probeResp = Client.send(probeReq, HttpResponse.BodyHandlers.ofString()); + total = new ObjectMapper().readTree(probeResp.body()).get("recordsTotal").asInt(); + System.out.println("tatal: " + total); + } catch (Exception e) { + e.printStackTrace(); + } + // 构建API请求URL，包含分页参数 + int want = Math.min(pageSize, total); + // String url = String.format(ExploitDbController.URL + "search?draw=1&start=%d&length=%d", start, pageSize); - // 发送HTTP请求并获取响应 - HttpResponse Response = Client.send(Request, HttpResponse.BodyHandlers.ofString()); - - // 调试模式下打印响应预览 - if (debug) { - System.out.println("响应内容预览：" + Response.body().substring(0, 500)); - } - - // 解析JSON响应 - JsonNode Root = this.Mapper.readTree(Response.body()); - JsonNode Data = Root.get("data"); - - // 处理数据数组 - if (Data.isArray()) { - // 确保数据有效性 - assert Data.isArray() && Data.size() > 0 : "Data 不是一个数组或大小为0"; - - // 输出进度信息 - System.out.println("正在爬取 Exploit-DB..."); - System.out.println("共找到 " + Data.size() + " 条记录。"); - System.out.println("正在写入数据库..."); + // 初始化状态码和结果列表 + int Status = 0; + List exploits = new ArrayList<>(); - // 调试模式下打印完整数据 + final int PAGE = 100; + for (int end = total; end > 0 && exploits.size() < want; end -= PAGE) { + int start = Math.max(Math.max(end - PAGE, 0), end - want); + int len = Math.min(PAGE, want - exploits.size()); + String url = String.format(URL + "search?draw=1&start=%d&length=%d", start, len); + try { + // 构建HTTP请求，设置必要的请求头 + HttpRequest Request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36") + .header("Accept", "application/json, text/javascript, */*; q=0.01") + .header("Accept-Language", "en-US,en;q=0.9") + .header("Referer", ExploitDbController.URL) + .header("X-Requested-With", "XMLHttpRequest") + .GET() + .build(); + + // 发送HTTP请求并获取响应 + HttpResponse Response = Client.send(Request, HttpResponse.BodyHandlers.ofString()); + + // 调试模式下打印响应预览 if (debug) { - System.out.println(Data); + System.out.println("响应内容预览：" + Response.body().substring(0, 500)); } - // 遍历每条漏洞记录并解析 - for (JsonNode Node : Data) { - Exploit exploit = new Exploit(); + // 解析JSON响应 + JsonNode Root = this.Mapper.readTree(Response.body()); + JsonNode Data = Root.get("data"); - // 设置漏洞ID - exploit.setId(Node.get("id").asText()); + // 处理数据数组 + if (Data.isArray()) { + // 确保数据有效性 + assert Data.isArray() && Data.size() > 0 : "Data 不是一个数组或大小为0"; - // 获取各字段节点 - JsonNode descriptionNode = Node.get("description"); - JsonNode authorNode = Node.get("author"); - JsonNode typeNode = Node.get("type"); - JsonNode platformNode = Node.get("platform"); + // 输出进度信息 + System.out.println("正在爬取 Exploit-DB..."); + System.out.println("共找到 " + Data.size() + " 条记录。"); + System.out.println("正在写入数据库..."); - // 解析描述信息（数组格式） - if (descriptionNode != null && descriptionNode.isArray() && descriptionNode.size() > 1) { - exploit.setDescription(descriptionNode.get(1).asText()); - } else { - exploit.setDescription(""); + // 调试模式下打印完整数据 + if (debug) { + System.out.println(Data); } - // 解析发布日期 - if (Node.has("date_published")) { - exploit.setDate(Node.get("date_published").asText()); - } else { - exploit.setDate(""); - } + // 遍历每条漏洞记录并解析 + for (JsonNode Node : Data) { + Exploit exploit = new Exploit(); + + // 设置漏洞ID + exploit.setId(Node.get("id").asText()); + + // 获取各字段节点 + JsonNode descriptionNode = Node.get("description"); + JsonNode authorNode = Node.get("author"); + JsonNode typeNode = Node.get("type"); + JsonNode platformNode = Node.get("platform"); + // 添加获取CVE编号的节点 + JsonNode codeNode = Node.get("code"); + + // 解析描述信息（数组格式） + if (descriptionNode != null && descriptionNode.isArray() && descriptionNode.size() > 1) { + exploit.setDescription(descriptionNode.get(1).asText()); + } else { + exploit.setDescription(""); + } - // 解析作者信息 - if (authorNode != null && authorNode.has("name")) { - exploit.setAuthor(authorNode.get("name").asText()); - } else { - exploit.setAuthor(""); - } + // 解析发布日期 + if (Node.has("date_published")) { + exploit.setDate(Node.get("date_published").asText()); + } else { + exploit.setDate(""); + } - // 解析漏洞类型 - if (typeNode != null && typeNode.has("name")) { - exploit.setType(typeNode.get("name").asText()); - } else { - exploit.setType(""); - } + // 解析作者信息 + if (authorNode != null && authorNode.has("name")) { + exploit.setAuthor(authorNode.get("name").asText()); + } else { + exploit.setAuthor(""); + } - // 解析影响平台 - if (platformNode != null && platformNode.has("platform")) { - exploit.setPlatform(platformNode.get("platform").asText()); - } else { - exploit.setPlatform(""); - } + // 解析漏洞类型 + if (typeNode != null && typeNode.has("name")) { + exploit.setType(typeNode.get("name").asText()); + } else { + exploit.setType(""); + } - // 添加到结果列表 - exploits.add(exploit); - } - // 将爬取的数据保存到数据库 - Status = super.AddExploit(exploits); - - // 如果数据保存成功 - if (Status != (-1)) { - // 在控制台打印数据表格 - ExploitDbController.PrintExploitTable(exploits); - - // 如果需要导出数据 - if (isExport) { - int status; - - // 根据指定的文件格式导出数据 - switch (extension) { - case "sql": - status = super.ExportToSQL(exploits); - break; - case "json": - status = super.ExportToJSON(exploits); - break; - case "yaml": - status = super.ExportToYAML(exploits); - break; - case "csv": - default: - status = super.ExportToCSV(exploits); - break; + // 解析影响平台 + if (platformNode != null && platformNode.has("platform")) { + exploit.setPlatform(platformNode.get("platform").asText()); + } else { + exploit.setPlatform(""); } - // 显示导出状态 - this.ExportStatus(status, extension); + // 添加到结果列表 + exploits.add(exploit); + if (exploits.size() >= want) + break; } } + Thread.sleep(1000); + } catch (IOException | InterruptedException e) { + String msg = e.getMessage(); + + if (msg != null && BLOCK_KEYWORDS.stream().anyMatch(k -> msg.toLowerCase().contains(k))) { + System.err.println("爬取 Exploit-DB : ⚠️ 请求被阻止，可能遭遇防火墙、访问限制或代理问题，请检查网络配置。"); + } else { + System.err.println("爬取 Exploit-DB 时出错：" + msg); + } } + } - } catch (IOException | InterruptedException e) { - String msg = e.getMessage(); + // 将爬取的数据保存到数据库 + Status = super.AddExploit(exploits); + + // 如果数据保存成功 + if (Status != (-1)) { + // 在控制台打印数据表格 + ExploitDbController.PrintExploitTable(exploits); + + // 如果需要导出数据 + if (isExport) { + int status; + + // 根据指定的文件格式导出数据 + switch (extension) { + case "sql": + status = super.ExportToSQL(exploits); + break; + case "json": + status = super.ExportToJSON(exploits); + break; + case "yaml": + status = super.ExportToYAML(exploits); + break; + case "csv": + default: + status = super.ExportToCSV(exploits); + break; + } - if (msg != null && BLOCK_KEYWORDS.stream().anyMatch(k -> msg.toLowerCase().contains(k))) { - System.err.println("爬取 Exploit-DB : ⚠️ 请求被阻止，可能遭遇防火墙、访问限制或代理问题，请检查网络配置。"); - } else { - System.err.println("爬取 Exploit-DB 时出错：" + msg); + // 显示导出状态 + this.ExportStatus(status, extension); } }