201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-27 01:30:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

爬虫代码优化

Browse Source
This commit is contained in:
xiongxiaoyang 2020-12-24 01:17:52 +08:00
parent 1b6cc8ccd5
commit c3daaecaaa
2 changed files with 14 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java
View File

@ -172,9 +172,6 @@ public class CrawlParser {
//总字数 //总字数
Integer totalWordCount = 0; Integer totalWordCount = 0;
//最新目录
Long lastIndexId = null;
String lastIndexName = null;
while (isFindIndex) { while (isFindIndex) {
@ -236,6 +233,9 @@ public class CrawlParser {
//章节更新 //章节更新
bookIndex.setId(hasIndex.getId()); bookIndex.setId(hasIndex.getId());
bookContent.setIndexId(hasIndex.getId()); bookContent.setIndexId(hasIndex.getId());
//计算总字数
totalWordCount = (totalWordCount+wordCount-hasIndex.getWordCount());
} else { } else {
//章节插入 //章节插入
//设置目录和章节内容 //设置目录和章节内容
@ -246,17 +246,12 @@ public class CrawlParser {
bookIndex.setCreateTime(currentDate); bookIndex.setCreateTime(currentDate);
bookContent.setIndexId(indexId); bookContent.setIndexId(indexId);
}
bookIndex.setUpdateTime(currentDate);
//判断是新书入库还是老书更新
if (hasIndexs.size() == 0) {
//新书入库
lastIndexId = bookIndex.getId();
lastIndexName = indexName;
//计算总字数 //计算总字数
totalWordCount += wordCount; totalWordCount += wordCount;
} }
bookIndex.setUpdateTime(currentDate);
} }
@ -267,18 +262,17 @@ public class CrawlParser {
isFindIndex = indexIdMatch.find() & indexNameMatch.find(); isFindIndex = indexIdMatch.find() & indexNameMatch.find();
} }
//判断是新书入库还是老书更新
if (hasIndexs.size() == 0) {
//新书入库
//设置小说基础信息 if (indexList.size() > 0) {
book.setWordCount(totalWordCount); //如果有爬到最新章节，则设置小说主表的最新章节信息
book.setLastIndexId(lastIndexId); //获取爬取到的最新章节
book.setLastIndexName(lastIndexName); BookIndex lastIndex = indexList.get(indexList.size()-1);
book.setLastIndexId(lastIndex.getId());
book.setLastIndexName(lastIndex.getIndexName());
book.setLastIndexUpdateTime(currentDate); book.setLastIndexUpdateTime(currentDate);
book.setCreateTime(currentDate);
} }
book.setWordCount(totalWordCount);
book.setUpdateTime(currentDate); book.setUpdateTime(currentDate);
if (indexList.size() == contentList.size() && indexList.size() > 0) { if (indexList.size() == contentList.size() && indexList.size() > 0) {

15
novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -80,6 +80,7 @@ public class BookServiceImpl implements BookService {
//保存小说主表 //保存小说主表
book.setCreateTime(new Date());
bookMapper.insertSelective(book); bookMapper.insertSelective(book);
//批量保存目录和内容 //批量保存目录和内容
@ -104,7 +105,7 @@ public class BookServiceImpl implements BookService {
@Override @Override
public Map<Integer, BookIndex> queryExistBookIndexMap(Long bookId) { public Map<Integer, BookIndex> queryExistBookIndexMap(Long bookId) {
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id,BookIndexDynamicSqlSupport.indexNum,BookIndexDynamicSqlSupport.indexName) List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id,BookIndexDynamicSqlSupport.indexNum,BookIndexDynamicSqlSupport.indexName,BookIndexDynamicSqlSupport.wordCount)
.from(BookIndexDynamicSqlSupport.bookIndex) .from(BookIndexDynamicSqlSupport.bookIndex)
.where(BookIndexDynamicSqlSupport.bookId,isEqualTo(bookId)) .where(BookIndexDynamicSqlSupport.bookId,isEqualTo(bookId))
.build() .build()
@ -118,7 +119,6 @@ public class BookServiceImpl implements BookService {
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
@Override @Override
public void updateBookAndIndexAndContent(Book book, List<BookIndex> bookIndexList, List<BookContent> bookContentList, Map<Integer, BookIndex> existBookIndexMap) { public void updateBookAndIndexAndContent(Book book, List<BookIndex> bookIndexList, List<BookContent> bookContentList, Map<Integer, BookIndex> existBookIndexMap) {
Date currentDate = new Date();
for (int i = 0; i < bookIndexList.size(); i++) { for (int i = 0; i < bookIndexList.size(); i++) {
BookIndex bookIndex = bookIndexList.get(i); BookIndex bookIndex = bookIndexList.get(i);
BookContent bookContent = bookContentList.get(i); BookContent bookContent = bookContentList.get(i);
@ -143,17 +143,6 @@ public class BookServiceImpl implements BookService {
} }
//更新小说主表 //更新小说主表
if(bookIndexList.size()>0) {
//有更新章节，才需要更新以下字段
book.setWordCount(queryTotalWordCount(book.getId()));
BookIndex lastIndex = bookIndexList.get(bookIndexList.size()-1);
if(!existBookIndexMap.containsKey(lastIndex.getIndexNum())) {
//如果最新章节不在已存在章节中，那么更新小说表最新章节信息
book.setLastIndexId(lastIndex.getId());
book.setLastIndexName(lastIndex.getIndexName());
book.setLastIndexUpdateTime(currentDate);
}
}
book.setBookName(null); book.setBookName(null);
book.setAuthorName(null); book.setAuthorName(null);
if(Constants.VISIT_COUNT_DEFAULT.equals(book.getVisitCount())) { if(Constants.VISIT_COUNT_DEFAULT.equals(book.getVisitCount())) {