201206030/novel
1
0
Fork 0
mirror of https://github.com/201206030/novel.git synced 2025-06-23 23:58:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

新增顶点小说网站解析，设置默认爬虫源为定点小说

Browse Source
This commit is contained in:
xiongxiaoyang
2019-12-18 14:04:21 +08:00
parent c71a5ebd6f
commit 793d32b314
7 changed files with 53 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqudaoConfig.java
View File

@ -19,7 +19,7 @@ public class CrawlBiqudaoConfig {
@Bean
@ConfigurationProperties(prefix = "biqudao.crawlsource") // prefix值必须是application.yml中对应属性的前缀
@ConditionalOnProperty(prefix = "biqudao.crawlsource",name = "enabled",havingValue = "true")
public BaseHtmlCrawlSource BiqutaCrawlSource() {
public BaseHtmlCrawlSource biqudaoCrawlSource() {
return new BiquCrawlSource();
}

4
novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqutaConfig.java
View File

@ -5,7 +5,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import xyz.zinglizingli.books.core.crawl.BaseHtmlCrawlSource;
import xyz.zinglizingli.books.core.crawl.BiquCrawlSource;
@ -18,10 +17,9 @@ public class CrawlBiqutaConfig {
@Bean
@Primary //必须加此注解，不然报错，下一个类则不需要添加
@ConfigurationProperties(prefix = "biquta.crawlsource") // prefix值必须是application.yml中对应属性的前缀
@ConditionalOnProperty(prefix = "biquta.crawlsource",name = "enabled",havingValue = "true")
public BaseHtmlCrawlSource BiqutaCrawlSource() {
public BaseHtmlCrawlSource biqutaCrawlSource() {
return new BiquCrawlSource();
}

29
novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlDingdianConfig.java Normal file
View File

@ -0,0 +1,29 @@
package xyz.zinglizingli.books.core.config;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import xyz.zinglizingli.books.core.crawl.BaseHtmlCrawlSource;
import xyz.zinglizingli.books.core.crawl.BiquCrawlSource;
/**
* @author 11797
*/
@Slf4j
@Configuration
public class CrawlDingdianConfig {
@Bean
@Primary //必须加此注解，不然报错，下一个类则不需要添加
@ConfigurationProperties(prefix = "dingdian.crawlsource") // prefix值必须是application.yml中对应属性的前缀
@ConditionalOnProperty(prefix = "dingdian.crawlsource",name = "enabled",havingValue = "true")
public BaseHtmlCrawlSource dingdianCrawlSource() {
return new BiquCrawlSource();
}
}

2
novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java
View File

@ -141,7 +141,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
//查询章节内容
String body3 = RestTemplateUtil.getBodyByUtf8(contentUrl.replace("//m.","//www."));
String body3 = RestTemplateUtil.getBodyByUtf8(contentUrl.replace("//m.","//www.").replace("//wap.","//www."));
if (body3 != null) {
String start = "id=\"content\">";
String end = "<script>";

21
novel-front/src/main/resources/application-crawl.yml
View File

@ -1,4 +1,4 @@
#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔 更多网站解析中，敬请期待
#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔, 3:顶点 更多网站解析中，敬请期待
biquta:
crawlsource:
enabled: true #是否开启此爬虫源
@ -30,4 +30,21 @@ biqudao:
pic-pattern: <img src="([^>]+)"\s+onerror="this.src=
intro-pattern: class="review">([^<]+)</p>
catalog-url-pattern: <a\s+href="(/bqge\d+/all\.html)">查看完整目录</a>
catalog-pattern: <a[^/]+style[^/]+href="(/bqge\d+/\d+\.html)">([^/]+)</a>
catalog-pattern: <a[^/]+style[^/]+href="(/bqge\d+/\d+\.html)">([^/]+)</a>
dingdian:
crawlsource:
enabled: true #是否开启此爬虫源
index-url: https://wap.dingdiann.com
list-page-url: https://wap.dingdiann.com/sort/{0}/{1}.html
book-url-pattern: href="/(ddk\d+)/"
score-pattern: <div\s+class="score">(\d+\.\d+)分</div>
book-name-pattern: <p class="title">([^/]+)</p>
author-pattern: 作者：([^/]+)<
status-pattern: 状态：([^/]+)</li>
cat-pattern: 类别：([^/]+)</li>
update-time-pattern: 更新：(\d+-\d+-\d+\s\d+:\d+:\d+)</a>
pic-pattern: <img src="([^>]+)"\s+onerror="this.src=
intro-pattern: class="review">([^<]+)</p>
catalog-url-pattern: <a\s+href="(/ddk\d+/all.html)">查看完整目录</a>
catalog-pattern: <a\s+style=""\s+href="(/ddk\d+/\d+\.html)">([^/]+)</a>