201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-27 01:30:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

爬虫代码优化

Browse Source
This commit is contained in:
xiongxiaoyang 2020-12-24 01:17:52 +08:00
parent 1b6cc8ccd5
commit c3daaecaaa
2 changed files with 14 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java
View File

@ -172,9 +172,6 @@ public class CrawlParser {
//总字数
Integer totalWordCount = 0;
//最新目录
Long lastIndexId = null;
String lastIndexName = null;
while (isFindIndex) {
@ -236,6 +233,9 @@ public class CrawlParser {
//章节更新
bookIndex.setId(hasIndex.getId());
bookContent.setIndexId(hasIndex.getId());
//计算总字数
totalWordCount = (totalWordCount+wordCount-hasIndex.getWordCount());
} else {
//章节插入
//设置目录和章节内容
@ -246,17 +246,12 @@ public class CrawlParser {
bookIndex.setCreateTime(currentDate);
bookContent.setIndexId(indexId);
}
bookIndex.setUpdateTime(currentDate);
//判断是新书入库还是老书更新
if (hasIndexs.size() == 0) {
//新书入库
lastIndexId = bookIndex.getId();
lastIndexName = indexName;
//计算总字数
totalWordCount += wordCount;
}
bookIndex.setUpdateTime(currentDate);
}
@ -267,18 +262,17 @@ public class CrawlParser {
isFindIndex = indexIdMatch.find() & indexNameMatch.find();
}
//判断是新书入库还是老书更新
if (hasIndexs.size() == 0) {
//新书入库
//设置小说基础信息
book.setWordCount(totalWordCount);
book.setLastIndexId(lastIndexId);
book.setLastIndexName(lastIndexName);
if (indexList.size() > 0) {
//如果有爬到最新章节，则设置小说主表的最新章节信息
//获取爬取到的最新章节
BookIndex lastIndex = indexList.get(indexList.size()-1);
book.setLastIndexId(lastIndex.getId());
book.setLastIndexName(lastIndex.getIndexName());
book.setLastIndexUpdateTime(currentDate);
book.setCreateTime(currentDate);
}
book.setWordCount(totalWordCount);
book.setUpdateTime(currentDate);
if (indexList.size() == contentList.size() && indexList.size() > 0) {

15
novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -80,6 +80,7 @@ public class BookServiceImpl implements BookService {
//保存小说主表
book.setCreateTime(new Date());
bookMapper.insertSelective(book);
//批量保存目录和内容
@ -104,7 +105,7 @@ public class BookServiceImpl implements BookService {
@Override
public Map<Integer, BookIndex> queryExistBookIndexMap(Long bookId) {
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id,BookIndexDynamicSqlSupport.indexNum,BookIndexDynamicSqlSupport.indexName)
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id,BookIndexDynamicSqlSupport.indexNum,BookIndexDynamicSqlSupport.indexName,BookIndexDynamicSqlSupport.wordCount)
.from(BookIndexDynamicSqlSupport.bookIndex)
.where(BookIndexDynamicSqlSupport.bookId,isEqualTo(bookId))
.build()
@ -118,7 +119,6 @@ public class BookServiceImpl implements BookService {
@Transactional(rollbackFor = Exception.class)
@Override
public void updateBookAndIndexAndContent(Book book, List<BookIndex> bookIndexList, List<BookContent> bookContentList, Map<Integer, BookIndex> existBookIndexMap) {
Date currentDate = new Date();
for (int i = 0; i < bookIndexList.size(); i++) {
BookIndex bookIndex = bookIndexList.get(i);
BookContent bookContent = bookContentList.get(i);
@ -143,17 +143,6 @@ public class BookServiceImpl implements BookService {
}
//更新小说主表
if(bookIndexList.size()>0) {
//有更新章节，才需要更新以下字段
book.setWordCount(queryTotalWordCount(book.getId()));
BookIndex lastIndex = bookIndexList.get(bookIndexList.size()-1);
if(!existBookIndexMap.containsKey(lastIndex.getIndexNum())) {
//如果最新章节不在已存在章节中，那么更新小说表最新章节信息
book.setLastIndexId(lastIndex.getId());
book.setLastIndexName(lastIndex.getIndexName());
book.setLastIndexUpdateTime(currentDate);
}
}
book.setBookName(null);
book.setAuthorName(null);
if(Constants.VISIT_COUNT_DEFAULT.equals(book.getVisitCount())) {