From 793d32b31476f50791f7761b45908bd5d3704fb2 Mon Sep 17 00:00:00 2001 From: xiongxiaoyang <773861846@qq.com> Date: Wed, 18 Dec 2019 14:04:21 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A1=B6=E7=82=B9=E5=B0=8F?= =?UTF-8?q?=E8=AF=B4=E7=BD=91=E7=AB=99=E8=A7=A3=E6=9E=90=EF=BC=8C=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E9=BB=98=E8=AE=A4=E7=88=AC=E8=99=AB=E6=BA=90=E4=B8=BA?= =?UTF-8?q?=E5=AE=9A=E7=82=B9=E5=B0=8F=E8=AF=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../books/core/config/CrawlBiqudaoConfig.java | 2 +- .../books/core/config/CrawlBiqutaConfig.java | 4 +-- .../core/config/CrawlDingdianConfig.java | 29 ++++++++++++++++++ .../books/core/crawl/BiquCrawlSource.java | 2 +- .../src/main/resources/application-crawl.yml | 21 +++++++++++-- script/crawlbook/application.yml | 4 +-- script/crawlbook/crawl-book-1.0-SNAPSHOT.jar | Bin 24676894 -> 24677598 bytes 7 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlDingdianConfig.java diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqudaoConfig.java b/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqudaoConfig.java index aa20164..4c207cb 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqudaoConfig.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqudaoConfig.java @@ -19,7 +19,7 @@ public class CrawlBiqudaoConfig { @Bean @ConfigurationProperties(prefix = "biqudao.crawlsource") // prefix值必须是application.yml中对应属性的前缀 @ConditionalOnProperty(prefix = "biqudao.crawlsource",name = "enabled",havingValue = "true") - public BaseHtmlCrawlSource BiqutaCrawlSource() { + public BaseHtmlCrawlSource biqudaoCrawlSource() { return new BiquCrawlSource(); } diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqutaConfig.java b/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqutaConfig.java index 5e21c2f..d149268 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqutaConfig.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiqutaConfig.java @@ -5,7 +5,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; import xyz.zinglizingli.books.core.crawl.BaseHtmlCrawlSource; import xyz.zinglizingli.books.core.crawl.BiquCrawlSource; @@ -18,10 +17,9 @@ public class CrawlBiqutaConfig { @Bean - @Primary //必须加此注解,不然报错,下一个类则不需要添加 @ConfigurationProperties(prefix = "biquta.crawlsource") // prefix值必须是application.yml中对应属性的前缀 @ConditionalOnProperty(prefix = "biquta.crawlsource",name = "enabled",havingValue = "true") - public BaseHtmlCrawlSource BiqutaCrawlSource() { + public BaseHtmlCrawlSource biqutaCrawlSource() { return new BiquCrawlSource(); } diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlDingdianConfig.java b/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlDingdianConfig.java new file mode 100644 index 0000000..17fc2ce --- /dev/null +++ b/novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlDingdianConfig.java @@ -0,0 +1,29 @@ +package xyz.zinglizingli.books.core.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import xyz.zinglizingli.books.core.crawl.BaseHtmlCrawlSource; +import xyz.zinglizingli.books.core.crawl.BiquCrawlSource; + +/** + * @author 11797 + */ +@Slf4j +@Configuration +public class CrawlDingdianConfig { + + + @Bean + @Primary //必须加此注解,不然报错,下一个类则不需要添加 + @ConfigurationProperties(prefix = "dingdian.crawlsource") // prefix值必须是application.yml中对应属性的前缀 + @ConditionalOnProperty(prefix = "dingdian.crawlsource",name = "enabled",havingValue = "true") + public BaseHtmlCrawlSource dingdianCrawlSource() { + return new BiquCrawlSource(); + } + + +} diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java b/novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java index 001f8af..ec567a2 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java @@ -141,7 +141,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource { //查询章节内容 - String body3 = RestTemplateUtil.getBodyByUtf8(contentUrl.replace("//m.","//www.")); + String body3 = RestTemplateUtil.getBodyByUtf8(contentUrl.replace("//m.","//www.").replace("//wap.","//www.")); if (body3 != null) { String start = "id=\"content\">"; String end = "