201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-05-16 23:08:30 +00:00
Code Issues Packages Releases Wiki Activity

perf(novel-crawl): 减小失效爬虫的CPU占用

Browse Source
This commit is contained in:
xiongxiaoyang 2025-05-12 18:18:48 +08:00
parent 55d5deea74
commit a4d6272a4f
1 changed files with 11 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -34,6 +34,7 @@ import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider; import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.time.Duration;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@ -308,12 +309,18 @@ public class CrawlServiceImpl implements CrawlService {
} catch (Exception e) { } catch (Exception e) {
log.error(e.getMessage(), e); log.error(e.getMessage(), e);
} }
if (page == totalPage) { if (page >= totalPage) {
// 第一遍采集完成，翻到第一页，继续第二次采集，适用于分页数比较少的最近更新列表 // 第一遍采集完成，翻到第一页，继续第二次采集，适用于分页数比较少的最近更新列表
page = 0; page = 1;
try {
// 第一遍采集完成，休眠1分钟
Thread.sleep(Duration.ofMinutes(1));
} catch (InterruptedException e) {
log.error(e.getMessage(), e);
}
}else{
page += 1;
} }
page += 1;
} }