Package com.bytedesk.kbase.llm_website.service

Class WebsiteCrawlerService

java.lang.Object

com.bytedesk.kbase.llm_website.service.WebsiteCrawlerService

@Service
public class WebsiteCrawlerService
extends Object

网站整站抓取服务 负责从指定的网站根域名开始，按照设定的抓取深度和规则，批量抓取整个网站内容

Field Summary

Fields

Modifier and Type	Field	Description
private final ConcurrentHashMap<String,WebsiteCrawlTask>	activeTasks
private final ExecutorService	crawlExecutor
private final WebsiteCrawlTaskRepository	crawlTaskRepository
private final UidUtils	uidUtils
private final WebpageRepository	webpageRepository
private final WebsiteRepository	websiteRepository

Constructor Summary

Constructors

Constructor	Description
WebsiteCrawlerService()

Method Summary

Modifier and Type	Method	Description
private boolean	crawlSinglePage(String url, WebsiteEntity website, Set<String> visitedUrls, Set<String> urlsToVisit, WebsiteCrawlConfig config)	抓取单个页面
private WebsiteCrawlTask	createCrawlTask(WebsiteEntity website, WebsiteCrawlConfig config)	创建抓取任务
private void	createOrUpdateWebpage(String url, String title, String description, String content, WebsiteEntity website)	创建或更新网页实体
private String	extractDescription(org.jsoup.nodes.Document doc)	提取页面描述
private void	extractLinks(org.jsoup.nodes.Document doc, String currentUrl, String baseUrl, Set<String> urlsToVisit, WebsiteCrawlConfig config)	提取页面链接
WebsiteCrawlTask	getCrawlTaskStatus(String taskId)	获取抓取任务状态
private boolean	isValidContent(String content, WebsiteCrawlConfig config)	验证内容是否有效
List<String>	parseSitemap(String sitemapUrl)	解析站点地图
private WebsiteCrawlResult	performCrawl(WebsiteEntity website, WebsiteCrawlTask task, WebsiteCrawlConfig config)	执行抓取任务
private String	resolveUrl(URL base, String href)	解析相对URL为绝对URL
private WebsiteCrawlTask	saveCrawlTask(WebsiteCrawlTask task)	保存抓取任务
private boolean	shouldCrawlUrl(String url, URL baseUrl, WebsiteCrawlConfig config)	判断URL是否应该被抓取
void	shutdown()	清理资源
CompletableFuture<WebsiteCrawlResult>	startCrawl(String websiteUid, WebsiteCrawlConfig config)	开始整站抓取
boolean	stopCrawlTask(String taskId)	停止抓取任务

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

websiteRepository

private final WebsiteRepository websiteRepository

webpageRepository

private final WebpageRepository webpageRepository

crawlTaskRepository

private final WebsiteCrawlTaskRepository crawlTaskRepository

uidUtils

private final UidUtils uidUtils

crawlExecutor

private final ExecutorService crawlExecutor

activeTasks

private final ConcurrentHashMap<String,WebsiteCrawlTask> activeTasks

Constructor Details

WebsiteCrawlerService

public WebsiteCrawlerService()

Method Details

startCrawl

@Async
public CompletableFuture<WebsiteCrawlResult> startCrawl(String websiteUid,
 WebsiteCrawlConfig config)

开始整站抓取

Parameters:

websiteUid - 网站UID

config - 抓取配置

Returns:

抓取任务

performCrawl

private WebsiteCrawlResult performCrawl(WebsiteEntity website,
 WebsiteCrawlTask task,
 WebsiteCrawlConfig config)

执行抓取任务

crawlSinglePage

private boolean crawlSinglePage(String url,
 WebsiteEntity website,
 Set<String> visitedUrls,
 Set<String> urlsToVisit,
 WebsiteCrawlConfig config)

抓取单个页面

extractDescription

private String extractDescription(org.jsoup.nodes.Document doc)

提取页面描述

extractLinks

private void extractLinks(org.jsoup.nodes.Document doc,
 String currentUrl,
 String baseUrl,
 Set<String> urlsToVisit,
 WebsiteCrawlConfig config)

提取页面链接

resolveUrl

private String resolveUrl(URL base,
 String href)

解析相对URL为绝对URL

shouldCrawlUrl

private boolean shouldCrawlUrl(String url,
 URL baseUrl,
 WebsiteCrawlConfig config)

判断URL是否应该被抓取

isValidContent

private boolean isValidContent(String content,
 WebsiteCrawlConfig config)

验证内容是否有效

createOrUpdateWebpage

private void createOrUpdateWebpage(String url,
 String title,
 String description,
 String content,
 WebsiteEntity website)

创建或更新网页实体

createCrawlTask

private WebsiteCrawlTask createCrawlTask(WebsiteEntity website,
 WebsiteCrawlConfig config)

创建抓取任务

saveCrawlTask

private WebsiteCrawlTask saveCrawlTask(WebsiteCrawlTask task)

保存抓取任务

getCrawlTaskStatus

public WebsiteCrawlTask getCrawlTaskStatus(String taskId)

获取抓取任务状态

stopCrawlTask

public boolean stopCrawlTask(String taskId)

停止抓取任务

parseSitemap

public List<String> parseSitemap(String sitemapUrl)

解析站点地图

shutdown

public void shutdown()

清理资源