diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java index 8c20655..a08fd3d 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java @@ -1,7 +1,5 @@ package com.java2nb.novel.core.crawl; -import com.java2nb.novel.core.cache.CacheKey; -import com.java2nb.novel.core.cache.CacheService; import com.java2nb.novel.core.utils.RandomBookInfoUtil; import com.java2nb.novel.core.utils.StringUtil; import com.java2nb.novel.entity.Book; @@ -48,6 +46,13 @@ public class CrawlParser { return crawlTaskProgress.get(taskId); } + /** + * 移除爬虫任务进度 + */ + public void removeCrawlTaskProgress(Long taskId) { + crawlTaskProgress.remove(taskId); + } + public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler) throws InterruptedException { Book book = new Book(); @@ -310,7 +315,7 @@ public class CrawlParser { if (task != null) { // 更新采集进度 - crawlTaskProgress.put(task.getId(), indexNum + 1); + crawlTaskProgress.put(task.getId(), indexList.size()); } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java index 94a9a0d..ad0de60 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java @@ -197,11 +197,11 @@ public class CrawlServiceImpl implements CrawlService { for (CrawlSingleTask crawlSingleTask : pageBean.getList()) { if (crawlSingleTask.getTaskStatus() == 2 && crawlParser.getCrawlTaskProgress(crawlSingleTask.getId()) != null) { - // 如果排队中的任务有任务进度 - // 1.设置任务进度 - crawlSingleTask.setCrawlChapters(crawlParser.getCrawlTaskProgress(crawlSingleTask.getId())); - // 2.将排队中的任务状态修改成采集中 + // 如果排队中的任务有任务进度,将排队中的任务状态修改成采集中并设置任务进度 crawlSingleTask.setTaskStatus((byte) 3); + crawlSingleTask.setCrawlChapters(crawlParser.getCrawlTaskProgress(crawlSingleTask.getId())); + // 只会有一个任务在采集中 + break; } } return pageBean; @@ -241,17 +241,14 @@ public class CrawlServiceImpl implements CrawlService { task.setCrawlChapters(crawlParser.getCrawlTaskProgress(task.getId())); } crawlSingleTaskMapper.updateByPrimaryKeySelective(task); + // 删除任务进度 + crawlParser.removeCrawlTaskProgress(task.getId()); } @Override public CrawlSource getCrawlSource(Integer id) { - Optional opt = crawlSourceMapper.selectByPrimaryKey(id); - if (opt.isPresent()) { - CrawlSource crawlSource = opt.get(); - return crawlSource; - } - return null; + return crawlSourceMapper.selectByPrimaryKey(id).orElse(null); } @Override