201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-26 17:20:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

优化

Browse Source
This commit is contained in:
xiongxiaoyang 2024-09-21 11:10:44 +08:00
parent a8219253e9
commit f78a2a36cf
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -16,6 +16,7 @@ import com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport;
import com.java2nb.novel.mapper.CrawlSourceMapper; import com.java2nb.novel.mapper.CrawlSourceMapper;
import com.java2nb.novel.service.BookService; import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.CrawlService; import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.utils.CrawlHttpClient;
import com.java2nb.novel.vo.CrawlSingleTaskVO; import com.java2nb.novel.vo.CrawlSingleTaskVO;
import com.java2nb.novel.vo.CrawlSourceVO; import com.java2nb.novel.vo.CrawlSourceVO;
import io.github.xxyopen.model.page.PageBean; import io.github.xxyopen.model.page.PageBean;
@ -38,7 +39,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClientWithChrome;
import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*; import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*;
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo; import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.select.SelectDSL.select; import static org.mybatis.dynamic.sql.select.SelectDSL.select;
@ -63,6 +63,8 @@ public class CrawlServiceImpl implements CrawlService {
private final IdWorker idWorker = IdWorker.INSTANCE; private final IdWorker idWorker = IdWorker.INSTANCE;
private CrawlHttpClient crawlHttpClient;
@Override @Override
public void addCrawlSource(CrawlSource source) { public void addCrawlSource(CrawlSource source) {
@ -260,7 +262,7 @@ public class CrawlServiceImpl implements CrawlService {
.replace("{catId}", ruleBean.getCatIdRule().get("catId" + catId)) .replace("{catId}", ruleBean.getCatIdRule().get("catId" + catId))
.replace("{page}", page + ""); .replace("{page}", page + "");
String bookListHtml = getByHttpClientWithChrome(catBookListUrl); String bookListHtml = crawlHttpClient.get(catBookListUrl);
if (bookListHtml != null) { if (bookListHtml != null) {
Pattern bookIdPatten = Pattern.compile(ruleBean.getBookIdPatten()); Pattern bookIdPatten = Pattern.compile(ruleBean.getBookIdPatten());
Matcher bookIdMatcher = bookIdPatten.matcher(bookListHtml); Matcher bookIdMatcher = bookIdPatten.matcher(bookListHtml);