201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-27 01:30:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

增加自动更新线程的配置

Browse Source
This commit is contained in:
xxy 2020-05-08 07:49:41 +08:00
parent 0e2e610d18
commit c9428bf0e7
3 changed files with 52 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java
View File

@ -9,9 +9,11 @@ import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.entity.CrawlSource; import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.service.BookService; import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.CrawlService; import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.utils.Constants;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.time.DateUtils; import org.apache.commons.lang3.time.DateUtils;
import org.springframework.beans.factory.annotation.Value;
import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener; import javax.servlet.ServletContextListener;
@ -33,43 +35,54 @@ public class StarterListener implements ServletContextListener {
private final CrawlService crawlService; private final CrawlService crawlService;
@Value("${crawl.update.thread}")
private int updateThreadCount;
@Override @Override
public void contextInitialized(ServletContextEvent sce) { public void contextInitialized(ServletContextEvent sce) {
log.info("程序启动,开始执行自动更新线程。。。"); log.info("程序启动,开始执行自动更新线程。。。");
new Thread(() -> { for(int i = 0 ; i<updateThreadCount; i++) {
while (true) { new Thread(() -> {
try { while (true) {
//1.查询最新目录更新时间在一个月之内的前100条需要更新的数据 try {
Date currentDate = new Date(); //1.查询最新目录更新时间在一个月之内的前100条需要更新的数据
Date startDate = DateUtils.addDays(currentDate, -30); Date currentDate = new Date();
List<Book> bookList = bookService.queryNeedUpdateBook(startDate, 100); Date startDate = DateUtils.addDays(currentDate, -30);
for (Book needUpdateBook : bookList) { List<Book> bookList ;
try { synchronized (this) {
//查询爬虫源规则 bookList = bookService.queryNeedUpdateBook(startDate, 100);
CrawlSource source = crawlService.queryCrawlSource(needUpdateBook.getCrawlSourceId()); }
RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class); for (Book needUpdateBook : bookList) {
//解析小说基本信息 try {
Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId()); //查询爬虫源规则
//这里只做老书更新 CrawlSource source = crawlService.queryCrawlSource(needUpdateBook.getCrawlSourceId());
book.setId(needUpdateBook.getId()); RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
book.setPicUrl(needUpdateBook.getPicUrl()); //解析小说基本信息
//查询已存在的章节 Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
Map<Integer, BookIndex> existBookIndexMap = bookService.queryExistBookIndexMap(needUpdateBook.getId()); //这里只做老书更新
//解析章节目录 book.setId(needUpdateBook.getId());
Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(),book, ruleBean, existBookIndexMap); if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
bookService.updateBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY),existBookIndexMap); //本地图片则不更新
}catch (Exception e){ book.setPicUrl(null);
log.error(e.getMessage(), e); }
//查询已存在的章节
Map<Integer, BookIndex> existBookIndexMap = bookService.queryExistBookIndexMap(needUpdateBook.getId());
//解析章节目录
Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(), book, ruleBean, existBookIndexMap);
bookService.updateBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY), existBookIndexMap);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
} }
Thread.sleep(1000 * 60 * 10);
} catch (Exception e) {
log.error(e.getMessage(), e);
} }
Thread.sleep(1000 * 60 * 10);
} catch (Exception e) {
log.error(e.getMessage(), e);
} }
}).start();
} }
}).start();
} }
} }

4
novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -180,10 +180,6 @@ public class BookServiceImpl implements BookService {
if(Constants.VISIT_COUNT_DEFAULT.equals(book.getVisitCount())) { if(Constants.VISIT_COUNT_DEFAULT.equals(book.getVisitCount())) {
book.setVisitCount(null); book.setVisitCount(null);
} }
if(book.getPicUrl()!=null && book.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
//本地图片则不更新
book.setPicUrl(null);
}
bookMapper.updateByPrimaryKeySelective(book); bookMapper.updateByPrimaryKeySelective(book);
} }

10
novel-crawl/src/main/resources/application.yml
View File

@ -1,3 +1,4 @@
#端口号
server: server:
port: 8081 port: 8081
@ -5,9 +6,16 @@ spring:
profiles: profiles:
active: dev active: dev
#登录用户名密码
admin: admin:
username: admin username: admin
password: admin password: admin
#爬虫自动更新的线程数
#建议小说数量不多或者正在运行新书入库爬虫的情况下设置为1即可
#随着小说数量的增多可以逐渐增加，但建议不要超出CPU的线程数
crawl:
update:
thread: 1