From 0e2e6229cdf0b17d847535a08168a265162a898e Mon Sep 17 00:00:00 2001 From: xiongxiaoyang <773861846@qq.com> Date: Mon, 8 Jun 2020 16:54:58 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=B0=E6=BA=90=EF=BC=8C?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E7=AE=80=E4=BB=8B=E4=B8=AD=E7=9A=84=E7=89=B9?= =?UTF-8?q?=E6=AE=8A=E5=AD=97=E7=AC=A6=EF=BC=8C=E6=8F=90=E9=AB=98=E7=88=AC?= =?UTF-8?q?=E8=99=AB=E5=85=BC=E5=AE=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- novel-common/pom.xml | 2 +- novel-crawl/pom.xml | 2 +- .../java/com/java2nb/novel/core/crawl/CrawlParser.java | 8 ++++++-- novel-front/pom.xml | 2 +- pom.xml | 2 +- sql/20200608.sql | 2 ++ 6 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 sql/20200608.sql diff --git a/novel-common/pom.xml b/novel-common/pom.xml index a42e663..a761e8c 100644 --- a/novel-common/pom.xml +++ b/novel-common/pom.xml @@ -5,7 +5,7 @@ novel com.java2nb - 2.5.0 + 2.5.1 4.0.0 diff --git a/novel-crawl/pom.xml b/novel-crawl/pom.xml index 759e827..f5fdc87 100644 --- a/novel-crawl/pom.xml +++ b/novel-crawl/pom.xml @@ -5,7 +5,7 @@ novel com.java2nb - 2.5.0 + 2.5.1 4.0.0 diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java index a8fbe52..66e0cd1 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java @@ -96,8 +96,12 @@ public class CrawlParser { String desc = bookDetailHtml.substring(bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length()); desc = desc.substring(0, desc.indexOf(ruleBean.getDescEnd())); - //过滤掉简介中的a标签 - desc = desc.replaceAll("",""); + //过滤掉简介中的特殊标签 + desc = desc.replaceAll("","") + .replaceAll("","") + .replaceAll("

\\s*

","") + .replaceAll("

","") + .replaceAll("

","
"); //设置书籍简介 book.setBookDesc(desc); if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) { diff --git a/novel-front/pom.xml b/novel-front/pom.xml index e477ad0..0227160 100644 --- a/novel-front/pom.xml +++ b/novel-front/pom.xml @@ -5,7 +5,7 @@ novel com.java2nb - 2.5.0 + 2.5.1 4.0.0 diff --git a/pom.xml b/pom.xml index 9612117..1b20eb9 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.java2nb novel - 2.5.0 + 2.5.1 novel-common novel-front diff --git a/sql/20200608.sql b/sql/20200608.sql new file mode 100644 index 0000000..350ab86 --- /dev/null +++ b/sql/20200608.sql @@ -0,0 +1,2 @@ +INSERT INTO `crawl_source` (`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES +(6, '新笔趣阁', '{\n \"bookListUrl\": \"http://www.xbiquge.la/fenlei/{catId}_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"6\",\n \"catId6\": \"5\"\n },\n \"bookIdPatten\": \"\",\n \"pagePatten\": \"(\\\\d+)/\\\\d+\",\n \"totalPagePatten\": \"\\\\d+/(\\\\d+)\",\n \"bookDetailUrl\": \"http://www.xbiquge.la/{bookId}/\",\n \"bookNamePatten\": \"

([^/]+)

\",\n \"authorNamePatten\": \"者：([^/]+)

\",\n \"picUrlPatten\": \"src=\\\"(http://www.xbiquge.la/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"bookStatusRule\": {},\n \"descStart\": \"
\",\n \"descEnd\": \"
\",\n \"upadateTimePatten\": \"

最后更新：(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)

\",\n \"upadateTimeFormatPatten\": \"yyyy-MM-dd HH:mm:ss\",\n \"bookIndexUrl\": \"http://www.xbiquge.la/{bookId}/\",\n \"indexIdPatten\": \"[^/]+\",\n \"indexNamePatten\": \"([^/]+)\",\n \"bookContentUrl\": \"http://www.xbiquge.la/{bookId}/{indexId}.html\",\n \"contentStart\": \"
\",\n \"contentEnd\": \"

\"\n}', 0, '2020-05-23 22:46:58', '2020-05-23 22:46:58');