Package com.bytedesk.kbase.llm_website

Class WebsiteRestService

java.lang.Object

com.bytedesk.core.base.BaseRestService<T,TRequest,TResponse>

com.bytedesk.core.base.BaseRestServiceWithExport<WebsiteEntity,WebsiteRequest,WebsiteResponse,WebsiteExcel>

com.bytedesk.kbase.llm_website.WebsiteRestService

@Service
public class WebsiteRestService
extends BaseRestServiceWithExport<WebsiteEntity,WebsiteRequest,WebsiteResponse,WebsiteExcel>

Field Summary

Fields

Modifier and Type	Field	Description
private final WebsiteCrawlTaskRepository	crawlTaskRepository
private final KbaseRestService	kbaseRestService
private final org.modelmapper.ModelMapper	modelMapper
private final UidUtils	uidUtils
private final WebsiteCrawlerService	websiteCrawlerService
private final WebsiteRepository	websiteRepository

Fields inherited from class com.bytedesk.core.base.BaseRestService

authService

Constructor Summary

Constructors

Constructor	Description
WebsiteRestService()

Method Summary

Modifier and Type	Method	Description
WebsiteExcel	convertToExcel(WebsiteEntity website)	转换为Excel对象 子类必须实现此方法
WebsiteResponse	convertToResponse(WebsiteEntity entity)
WebsiteResponse	create(WebsiteRequest request)
protected org.springframework.data.jpa.domain.Specification<WebsiteEntity>	createSpecification(WebsiteRequest request)	创建Specification对象，子类必须实现 用于queryByOrg的通用实现
void	delete(WebsiteRequest request)
void	deleteAll(WebsiteRequest request)
void	deleteByUid(String uid)
protected WebsiteEntity	doSave(WebsiteEntity entity)	子类实现具体的保存逻辑
WebsiteResponse	enable(WebsiteRequest request)
protected org.springframework.data.domain.Page<WebsiteEntity>	executePageQuery(org.springframework.data.jpa.domain.Specification<WebsiteEntity> spec, org.springframework.data.domain.Pageable pageable)	执行分页查询，子类必须实现 用于queryByOrg的通用实现
List<WebsiteEntity>	findByKbUid(String kbUid)
Optional<WebsiteEntity>	findByUid(String uid)
WebsiteCrawlConfig	getCrawlConfig(String websiteUid)	获取网站抓取配置
List<WebsiteCrawlTask>	getCrawlTasks(String websiteUid)	获取网站的抓取任务列表
WebsiteCrawlTask	getCrawlTaskStatus(String taskId)	获取抓取任务状态
Optional<WebsiteCrawlTask>	getLatestCrawlTask(String websiteUid)	获取网站的最新抓取任务
WebsiteEntity	handleOptimisticLockingFailureException(org.springframework.orm.ObjectOptimisticLockingFailureException e, WebsiteEntity entity)
List<String>	parseSitemap(String websiteUid)	从sitemap开始抓取
CompletableFuture<WebsiteCrawlResult>	startCrawl(String websiteUid, WebsiteCrawlConfig config)	开始整站抓取
CompletableFuture<WebsiteCrawlResult>	startCrawlWithDefaultConfig(String websiteUid)	使用默认配置开始抓取
CompletableFuture<WebsiteCrawlResult>	startDeepCrawl(String websiteUid)	使用深度配置开始抓取
CompletableFuture<WebsiteCrawlResult>	startFastCrawl(String websiteUid)	使用快速配置开始抓取
boolean	stopCrawl(String websiteUid)	停止抓取任务
WebsiteResponse	update(WebsiteRequest request)
WebsiteResponse	updateCrawlConfig(String websiteUid, WebsiteCrawlConfig config)	更新网站抓取配置

Methods inherited from class com.bytedesk.core.base.BaseRestServiceWithExport

queryByOrgEntity

Methods inherited from class com.bytedesk.core.base.BaseRestService

deleteByOrgUid, findByOrgUid, getUidFromRequest, queryByOrg, queryByUid, queryByUser, recover, save, setUserUidToRequest

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

websiteRepository

private final WebsiteRepository websiteRepository

modelMapper

private final org.modelmapper.ModelMapper modelMapper

uidUtils

private final UidUtils uidUtils

kbaseRestService

private final KbaseRestService kbaseRestService

websiteCrawlerService

private final WebsiteCrawlerService websiteCrawlerService

crawlTaskRepository

private final WebsiteCrawlTaskRepository crawlTaskRepository

Constructor Details

WebsiteRestService

public WebsiteRestService()

Method Details

createSpecification

protected org.springframework.data.jpa.domain.Specification<WebsiteEntity> createSpecification(WebsiteRequest request)

Description copied from class: BaseRestService

创建Specification对象，子类必须实现 用于queryByOrg的通用实现

Specified by:

createSpecification in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

executePageQuery

protected org.springframework.data.domain.Page<WebsiteEntity> executePageQuery(org.springframework.data.jpa.domain.Specification<WebsiteEntity> spec,
 org.springframework.data.domain.Pageable pageable)

Description copied from class: BaseRestService

执行分页查询，子类必须实现 用于queryByOrg的通用实现

Specified by:

executePageQuery in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

findByUid

@Cacheable(value="website",
           key="#uid",
           unless="#result==null")
public Optional<WebsiteEntity> findByUid(String uid)

Specified by:

findByUid in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

findByKbUid

@Cacheable(value="website",
           key="#kbUid",
           unless="#result==null")
public List<WebsiteEntity> findByKbUid(String kbUid)

create

public WebsiteResponse create(WebsiteRequest request)

Specified by:

create in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

update

public WebsiteResponse update(WebsiteRequest request)

Specified by:

update in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

doSave

protected WebsiteEntity doSave(WebsiteEntity entity)

Description copied from class: BaseRestService

子类实现具体的保存逻辑

Specified by:

doSave in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

handleOptimisticLockingFailureException

public WebsiteEntity handleOptimisticLockingFailureException(org.springframework.orm.ObjectOptimisticLockingFailureException e,
 WebsiteEntity entity)

Specified by:

handleOptimisticLockingFailureException in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

deleteByUid

public void deleteByUid(String uid)

Specified by:

deleteByUid in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

delete

public void delete(WebsiteRequest request)

Specified by:

delete in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

deleteAll

public void deleteAll(WebsiteRequest request)

enable

public WebsiteResponse enable(WebsiteRequest request)

convertToResponse

public WebsiteResponse convertToResponse(WebsiteEntity entity)

Specified by:

convertToResponse in class BaseRestService<WebsiteEntity,WebsiteRequest,WebsiteResponse>

convertToExcel

public WebsiteExcel convertToExcel(WebsiteEntity website)

Description copied from class: BaseRestServiceWithExport

转换为Excel对象 子类必须实现此方法

Specified by:

convertToExcel in class BaseRestServiceWithExport<WebsiteEntity,WebsiteRequest,WebsiteResponse,WebsiteExcel>

startCrawl

public CompletableFuture<WebsiteCrawlResult> startCrawl(String websiteUid,
 WebsiteCrawlConfig config)

开始整站抓取

startCrawlWithDefaultConfig

public CompletableFuture<WebsiteCrawlResult> startCrawlWithDefaultConfig(String websiteUid)

使用默认配置开始抓取

startFastCrawl

public CompletableFuture<WebsiteCrawlResult> startFastCrawl(String websiteUid)

使用快速配置开始抓取

startDeepCrawl

public CompletableFuture<WebsiteCrawlResult> startDeepCrawl(String websiteUid)

使用深度配置开始抓取

stopCrawl

public boolean stopCrawl(String websiteUid)

停止抓取任务

getCrawlTaskStatus

public WebsiteCrawlTask getCrawlTaskStatus(String taskId)

获取抓取任务状态

getCrawlTasks

public List<WebsiteCrawlTask> getCrawlTasks(String websiteUid)

获取网站的抓取任务列表

getLatestCrawlTask

public Optional<WebsiteCrawlTask> getLatestCrawlTask(String websiteUid)

获取网站的最新抓取任务

parseSitemap

public List<String> parseSitemap(String websiteUid)

从sitemap开始抓取

updateCrawlConfig

public WebsiteResponse updateCrawlConfig(String websiteUid,
 WebsiteCrawlConfig config)

更新网站抓取配置

getCrawlConfig

public WebsiteCrawlConfig getCrawlConfig(String websiteUid)

获取网站抓取配置