201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-06-05 06:18:29 +00:00
Code Issues Packages Releases Wiki Activity

fix(novel-crawl): 解决爬虫进程间的冲突问题，支持同时启动多个爬虫进程

Browse Source
This commit is contained in:
xiongxiaoyang 2025-05-13 11:11:27 +08:00
parent a07643bde0
commit 42bcecc304
2 changed files with 4 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java vendored
View File

@ -41,11 +41,6 @@ public interface CacheKey {
* */ * */
String TEMPLATE_DIR_KEY = "templateDirKey";; String TEMPLATE_DIR_KEY = "templateDirKey";;
/**
* 正在运行的爬虫线程存储KEY前缀
* */
String RUNNING_CRAWL_THREAD_KEY_PREFIX = "runningCrawlTreadDataKeyPrefix";
/** /**
* 上一次搜索引擎更新的时间 * 上一次搜索引擎更新的时间
* */ * */

7
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -68,6 +68,8 @@ public class CrawlServiceImpl implements CrawlService {
private final Map<Integer, Byte> crawlSourceStatusMap = new HashMap<>(); private final Map<Integer, Byte> crawlSourceStatusMap = new HashMap<>();
private final Map<Integer, Set<Long>> runningCrawlThread = new HashMap<>();
@Override @Override
public void addCrawlSource(CrawlSource source) { public void addCrawlSource(CrawlSource source) {
@ -123,8 +125,7 @@ public class CrawlServiceImpl implements CrawlService {
if (sourceStatus == (byte) 0) { if (sourceStatus == (byte) 0) {
// 关闭 // 关闭
// 将该爬虫源正在运行的线程集合全部停止 // 将该爬虫源正在运行的线程集合全部停止
Set<Long> runningCrawlThreadId = (Set<Long>) cacheService.getObject( Set<Long> runningCrawlThreadId = runningCrawlThread.get(sourceId);
CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + sourceId);
if (runningCrawlThreadId != null) { if (runningCrawlThreadId != null) {
for (Long ThreadId : runningCrawlThreadId) { for (Long ThreadId : runningCrawlThreadId) {
Thread thread = ThreadUtil.findThread(ThreadId); Thread thread = ThreadUtil.findThread(ThreadId);
@ -152,7 +153,7 @@ public class CrawlServiceImpl implements CrawlService {
//thread加入到监控缓存中 //thread加入到监控缓存中
threadIds.add(thread.getId()); threadIds.add(thread.getId());
} }
cacheService.setObject(CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + sourceId, threadIds); runningCrawlThread.put(sourceId, threadIds);
} }
} }