201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-07-14 21:26:41 +00:00
Code Issues Packages Releases Wiki Activity

perf(novel-crawl): 优化单本采集任务进度显示功能

Browse Source
This commit is contained in:
xiongxiaoyang
2025-07-11 22:46:53 +08:00
parent 8c9013ad05
commit 8c572edb10
2 changed files with 15 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java
View File

@ -1,7 +1,5 @@
package com.java2nb.novel.core.crawl; package com.java2nb.novel.core.crawl;
import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.utils.RandomBookInfoUtil; import com.java2nb.novel.core.utils.RandomBookInfoUtil;
import com.java2nb.novel.core.utils.StringUtil; import com.java2nb.novel.core.utils.StringUtil;
import com.java2nb.novel.entity.Book; import com.java2nb.novel.entity.Book;
@ -48,6 +46,13 @@ public class CrawlParser {
return crawlTaskProgress.get(taskId); return crawlTaskProgress.get(taskId);
} }
/**
* 移除爬虫任务进度
*/
public void removeCrawlTaskProgress(Long taskId) {
crawlTaskProgress.remove(taskId);
}
public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler) public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler)
throws InterruptedException { throws InterruptedException {
Book book = new Book(); Book book = new Book();
@ -310,7 +315,7 @@ public class CrawlParser {
if (task != null) { if (task != null) {
// 更新采集进度 // 更新采集进度
crawlTaskProgress.put(task.getId(), indexNum + 1); crawlTaskProgress.put(task.getId(), indexList.size());
} }

17
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -197,11 +197,11 @@ public class CrawlServiceImpl implements CrawlService {
for (CrawlSingleTask crawlSingleTask : pageBean.getList()) { for (CrawlSingleTask crawlSingleTask : pageBean.getList()) {
if (crawlSingleTask.getTaskStatus() == 2 if (crawlSingleTask.getTaskStatus() == 2
&& crawlParser.getCrawlTaskProgress(crawlSingleTask.getId()) != null) { && crawlParser.getCrawlTaskProgress(crawlSingleTask.getId()) != null) {
// 如果排队中的任务有任务进度 // 如果排队中的任务有任务进度，将排队中的任务状态修改成采集中并设置任务进度
// 1.设置任务进度
crawlSingleTask.setCrawlChapters(crawlParser.getCrawlTaskProgress(crawlSingleTask.getId()));
// 2.将排队中的任务状态修改成采集中
crawlSingleTask.setTaskStatus((byte) 3); crawlSingleTask.setTaskStatus((byte) 3);
crawlSingleTask.setCrawlChapters(crawlParser.getCrawlTaskProgress(crawlSingleTask.getId()));
// 只会有一个任务在采集中
break;
} }
} }
return pageBean; return pageBean;
@ -241,17 +241,14 @@ public class CrawlServiceImpl implements CrawlService {
task.setCrawlChapters(crawlParser.getCrawlTaskProgress(task.getId())); task.setCrawlChapters(crawlParser.getCrawlTaskProgress(task.getId()));
} }
crawlSingleTaskMapper.updateByPrimaryKeySelective(task); crawlSingleTaskMapper.updateByPrimaryKeySelective(task);
// 删除任务进度
crawlParser.removeCrawlTaskProgress(task.getId());
} }
@Override @Override
public CrawlSource getCrawlSource(Integer id) { public CrawlSource getCrawlSource(Integer id) {
Optional<CrawlSource> opt = crawlSourceMapper.selectByPrimaryKey(id); return crawlSourceMapper.selectByPrimaryKey(id).orElse(null);
if (opt.isPresent()) {
CrawlSource crawlSource = opt.get();
return crawlSource;
}
return null;
} }
@Override @Override