mirror of
https://github.com/201206030/novel.git
synced 2025-06-14 03:38:31 +00:00
爬虫自动更新程序优化，增加自动修复错误章节
This commit is contained in:
xiongxiaoyang
parent
8f5f141316
commit
eac0ce9302
@ -522,10 +522,8 @@ public class BookCrawlServiceImpl implements BookCrawlService {
|
||||
if (picMather.find()) {
|
||||
String picSrc = picMather.group(1);
|
||||
|
||||
Pattern descPatten = compile("class=\"review\">([^<]+)</p>");
|
||||
Matcher descMatch = descPatten.matcher(body);
|
||||
if (descMatch.find()) {
|
||||
String desc = descMatch.group(1);
|
||||
String desc = body.substring(body.indexOf("<p class=\"review\">") + "<p class=\"review\">".length());
|
||||
desc = desc.substring(0, desc.indexOf("</p>"));
|
||||
|
||||
|
||||
BookDO book = new BookDO();
|
||||
@ -604,7 +602,6 @@ public class BookCrawlServiceImpl implements BookCrawlService {
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -857,9 +854,6 @@ public class BookCrawlServiceImpl implements BookCrawlService {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private String getByTemplate(String catBookListUrl) {
|
||||
try {
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(catBookListUrl, String.class);
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package xyz.zinglizingli.books.core.crawl;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import xyz.zinglizingli.books.po.Book;
|
||||
import xyz.zinglizingli.books.po.BookContent;
|
||||
@ -14,6 +15,7 @@ import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@ -94,12 +96,8 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
|
||||
Matcher picMather = picPatten.matcher(body);
|
||||
if (picMather.find()) {
|
||||
String picSrc = picMather.group(1);
|
||||
|
||||
|
||||
Pattern descPatten = compile(getIntroPattern());
|
||||
Matcher descMatch = descPatten.matcher(body);
|
||||
if (descMatch.find()) {
|
||||
String desc = descMatch.group(1);
|
||||
String desc = body.substring(body.indexOf("<p class=\"review\">") + "<p class=\"review\">".length());
|
||||
desc = desc.substring(0, desc.indexOf("</p>"));
|
||||
|
||||
|
||||
Book book = new Book();
|
||||
@ -130,15 +128,15 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
|
||||
int indexNum = 0;
|
||||
|
||||
//查询该书籍已存在目录号
|
||||
List<Integer> hasIndexNum = bookService.queryIndexNumByBookNameAndAuthor(bookName, author);
|
||||
//更新和插入分别开，插入只在凌晨做一次
|
||||
if (hasIndexNum.size() > 0) {
|
||||
Map<Integer, BookIndex> hasIndexs = bookService.queryIndexByBookNameAndAuthor(bookName, author);
|
||||
//更新和插入分别开，此处只做更新
|
||||
if (hasIndexs.size() > 0) {
|
||||
while (isFindIndex) {
|
||||
if (!hasIndexNum.contains(indexNum)) {
|
||||
|
||||
String contentUrl = getIndexUrl() + indexListMatch.group(1);
|
||||
BookIndex hasIndex = hasIndexs.get(indexNum);
|
||||
String indexName = indexListMatch.group(2);
|
||||
|
||||
if (hasIndex == null || !StringUtils.deleteWhitespace(hasIndex.getIndexName()).equals(StringUtils.deleteWhitespace(indexName))) {
|
||||
String contentUrl = getIndexUrl() + indexListMatch.group(1);
|
||||
|
||||
//查询章节内容
|
||||
String body3 = RestTemplateUtil.getBodyByUtf8(contentUrl.replace("//m.", "//www.").replace("//wap.", "//www."));
|
||||
@ -169,9 +167,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
|
||||
}
|
||||
|
||||
if (indexList.size() == contentList.size() && indexList.size() > 0) {
|
||||
ExcutorUtils.excuteFixedTask(() ->
|
||||
bookService.saveBookAndIndexAndContent(book, indexList, contentList)
|
||||
);
|
||||
bookService.saveBookAndIndexAndContent(book, indexList, contentList);
|
||||
|
||||
}
|
||||
}
|
||||
@ -186,7 +182,6 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -212,5 +207,4 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -30,6 +30,7 @@ import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
@ -109,13 +110,13 @@ public class BookService {
|
||||
newBookIndexList.add(bookIndexItem);
|
||||
newContentList.add(bookContentItem);
|
||||
}
|
||||
//一次最多只允许插入20条记录,否则影响服务器响应
|
||||
if (isUpdate && i % 20 == 0 && newBookIndexList.size() > 0) {
|
||||
//一次最多只允许插入100条记录,否则影响服务器响应
|
||||
if (isUpdate && i % 100 == 0 && newBookIndexList.size() > 0) {
|
||||
bookService.insertIndexListAndContentList(newBookIndexList, newContentList);
|
||||
newBookIndexList = new ArrayList<>();
|
||||
newContentList = new ArrayList<>();
|
||||
try {
|
||||
Thread.sleep(1000 * 60 * 5);
|
||||
Thread.sleep(1000 * 60 * 1);
|
||||
} catch (InterruptedException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
throw new RuntimeException(e.getMessage());
|
||||
@ -175,13 +176,28 @@ public class BookService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量插入章节目录表和章节内容表
|
||||
* 批量插入章节目录表和章节内容表（自动修复错误章节）
|
||||
* */
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void insertIndexListAndContentList(List<BookIndex> newBookIndexList, List<BookContent> newContentList) {
|
||||
long start = System.currentTimeMillis();
|
||||
if(newBookIndexList.size() > 0) {
|
||||
//删除已存在的错误章节
|
||||
List<Integer> indexNumberList = newBookIndexList.stream().map(BookIndex::getIndexNum).collect(Collectors.toList());
|
||||
Long bookId = newBookIndexList.get(0).getBookId();
|
||||
BookIndexExample bookIndexExample = new BookIndexExample();
|
||||
bookIndexExample.createCriteria().andBookIdEqualTo(bookId).andIndexNumIn(indexNumberList);
|
||||
bookIndexMapper.deleteByExample(bookIndexExample);
|
||||
BookContentExample bookContentExample = new BookContentExample();
|
||||
bookContentExample.createCriteria().andBookIdEqualTo(bookId).andIndexNumIn(indexNumberList);
|
||||
bookContentMapper.deleteByExample(bookContentExample);
|
||||
|
||||
//插入新的章节
|
||||
bookIndexMapper.insertBatch(newBookIndexList);
|
||||
bookContentMapper.insertBatch(newContentList);
|
||||
}
|
||||
log.info("更新章节耗时："+(System.currentTimeMillis()-start));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
@ -307,7 +323,7 @@ public class BookService {
|
||||
/**
|
||||
* 查询该书籍已存在目录号
|
||||
*/
|
||||
public List<Integer> queryIndexNumByBookNameAndAuthor(String bookName, String author) {
|
||||
public Map<Integer,BookIndex> queryIndexByBookNameAndAuthor(String bookName, String author) {
|
||||
BookExample example = new BookExample();
|
||||
example.createCriteria().andBookNameEqualTo(bookName).andAuthorEqualTo(author);
|
||||
List<Book> books = bookMapper.selectByExample(example);
|
||||
@ -318,12 +334,12 @@ public class BookService {
|
||||
bookIndexExample.createCriteria().andBookIdEqualTo(bookId);
|
||||
List<BookIndex> bookIndices = bookIndexMapper.selectByExample(bookIndexExample);
|
||||
if(bookIndices.size() > 0) {
|
||||
return bookIndices.stream().map(BookIndex::getIndexNum).collect(Collectors.toList());
|
||||
return bookIndices.stream().collect(Collectors.toMap(BookIndex::getIndexNum, Function.identity()));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return new ArrayList<>(0);
|
||||
return new HashMap<>(0);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -45,6 +45,6 @@ dingdian:
|
||||
cat-pattern: 类别：([^/]+)</li>
|
||||
update-time-pattern: 更新：(\d+-\d+-\d+\s\d+:\d+:\d+)</a>
|
||||
pic-pattern: <img src="([^>]+)"\s+onerror="this.src=
|
||||
intro-pattern: class="review">([^<]+)</p>
|
||||
intro-pattern: class="review">([^/]+)</p>
|
||||
catalog-url-pattern: <a\s+href="(/ddk\d+/all.html)">查看完整目录</a>
|
||||
catalog-pattern: <a\s+style=""\s+href="(/ddk\d+/\d+\.html)">([^/]+)</a>
|
||||
@ -9,6 +9,6 @@ mybatis:
|
||||
mysql: {charset: utf8mb4}
|
||||
books: {lowestScore: '8.5'}
|
||||
crawl:
|
||||
website: {type: '3'}
|
||||
website: {type: '2'}
|
||||
soft-novel: '0'
|
||||
manhua: '0'
|
||||
|
||||
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user