update

novel-front/pom.xml

@@ -10,7 +10,7 @@
     </parent>
     <groupId>xyz.zinglizingli</groupId>
     <artifactId>novel-front</artifactId>
-    <version>2.2.0.beta</version>
+    <version>2.3.0.beta</version>
     <name>novel-front</name>
     <description>小说精品楼-前台web网站</description>
 

novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBaishuzhaiConfig.java

@@ -0,0 +1,29 @@
+package xyz.zinglizingli.books.core.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import xyz.zinglizingli.books.core.crawl.BaseHtmlCrawlSource;
+import xyz.zinglizingli.books.core.crawl.BiquCrawlSource;
+
+/**
+ * @author 11797
+ */
+@Slf4j
+@Configuration
+public class CrawlBaishuzhaiConfig {
+
+
+    @Bean
+    @Primary //必须加此注解，不然报错，下一个类则不需要添加
+    @ConfigurationProperties(prefix = "baishuzhai.crawlsource") // prefix值必须是application.yml中对应属性的前缀
+    @ConditionalOnProperty(prefix = "crawl.website",name = "type",havingValue = "4")
+    public BaseHtmlCrawlSource dingdianCrawlSource() {
+        return new BiquCrawlSource();
+    }
+
+
+}

novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlDingdianConfig.java

@@ -18,7 +18,6 @@ public class CrawlDingdianConfig {
 
 
     @Bean
-    @Primary //必须加此注解，不然报错，下一个类则不需要添加
     @ConfigurationProperties(prefix = "dingdian.crawlsource") // prefix值必须是application.yml中对应属性的前缀
     @ConditionalOnProperty(prefix = "crawl.website",name = "type",havingValue = "3")
     public BaseHtmlCrawlSource dingdianCrawlSource() {

novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java

@@ -31,15 +31,8 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
     @Override
     public void parse() {
 
-        Map<Integer,Date> cat2Date = bookService.queryLastUpdateTime();
+        for(int page = 1; page<= Constants.UPDATE_PAGES_ONCE; page++) {
-        Map<Integer,Date> newCat2Date = new HashMap<>();
+            String catBookListUrl = getListPageUrl().replace("{0}", "0").replace("{1}", page+"");
-        for(int i=1;i<=7;i++) {
-            Date lastUpdateTime = cat2Date.get(i);
-            Date updateTime = lastUpdateTime;
-            int page = 1;
-            do{
-                String catBookListUrl = getListPageUrl().replace("{0}", "0").replace("{1}", page + "");
-                page++;
             String forObject = RestTemplateUtil.getBodyByUtf8(catBookListUrl);
             if (forObject != null) {
                 //解析第一页书籍的数据
@@ -56,13 +49,9 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                 Matcher bookNameMatch = bookNamePatten.matcher(forObject);
 
-                    Pattern authorPatten = compile(getAuthorPattern());
-
-                    Matcher authorMatch = authorPatten.matcher(forObject);
-
                 boolean isBookNameMatch = bookNameMatch.find();
 
-                    while (isFind && scoreFind && isBookNameMatch && authorMatch.find() && updateTime.getTime()>=lastUpdateTime.getTime()) {
+                while (isFind && scoreFind && isBookNameMatch) {
 
                     try {
                         Float score = Float.parseFloat(scoreMatch.group(1));
@@ -76,30 +65,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                         String bookName = bookNameMatch.group(1);
 
-                            String author = authorMatch.group(1);
-
-                            Boolean hasBook = bookService.hasBook(bookName, author);
-
-                            if (hasBook) {
-
                         bookService.addBookParseLog(bookUrl, bookName, score);
-                            }
-
-                            String body = RestTemplateUtil.getBodyByUtf8(bookUrl);
-                            if (body != null) {
-                                Pattern updateTimePatten = compile(getUpdateTimePattern());
-                                Matcher updateTimeMatch = updateTimePatten.matcher(body);
-                                if (updateTimeMatch.find()) {
-                                    String updateTimeStr = updateTimeMatch.group(1);
-                                    SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
-                                    updateTime = format.parse(updateTimeStr);
-                                    if(!newCat2Date.containsKey(i)) {
-                                        newCat2Date.put(i, updateTime);
-                                    }
-
-
-                                }
-                            }
 
 
                     } catch (Exception e) {
@@ -116,16 +82,13 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                 }
             }
-            }while (updateTime.getTime()>=lastUpdateTime.getTime());
         }
-        bookService.updateBookUpdateTimeLog(newCat2Date);
 
     }
 
     @Override
     public void update() {
         List<BookParseLog> logs = bookService.queryBookParseLogs();
-        List<Long> successLogIds = new ArrayList<>();
         for (BookParseLog bookParseLog : logs) {
             try {
 
@@ -158,10 +121,16 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                 Pattern updateTimePatten = compile(getUpdateTimePattern());
                                 Matcher updateTimeMatch = updateTimePatten.matcher(body);
-                                if (updateTimeMatch.find()) {
+                                /*if (updateTimeMatch.find()) {
                                     String updateTimeStr = updateTimeMatch.group(1);
-                                    SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
+                                    SimpleDateFormat format ;
-                                    Date updateTime = format.parse(updateTimeStr);
+                                    if(updateTimeStr.length()>10){
+
+                                        format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
+                                    }else{
+                                        format = new SimpleDateFormat("yy-MM-dd");
+                                    }
+                                    Date updateTime = format.parse(updateTimeStr);*/
                                     Pattern picPatten = compile(getPicPattern());
                                     Matcher picMather = picPatten.matcher(body);
                                     if (picMather.find()) {
@@ -178,7 +147,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
                                         book.setScore(score > 10 ? 8.0f : score);
                                         book.setPicUrl(picSrc);
                                         book.setBookStatus(status);
-                                        book.setUpdateTime(updateTime);
+                                        book.setUpdateTime(new Date());
 
                                         List<BookIndex> indexList = new ArrayList<>();
                                         List<BookContent> contentList = new ArrayList<>();
@@ -241,7 +210,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                                     }
                                                 }
-                                                successLogIds.add(bookParseLog.getId());
+                                                bookService.deleteBookParseLog(bookParseLog.getId());
                                             }
 
 
@@ -250,7 +219,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                     }
 
-                                }
+                                //}
                             }
                         }
 
@@ -266,7 +235,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
             }
         }
 
-        bookService.deleteBookParseLogs(successLogIds);
+
 
     }
 

novel-front/src/main/java/xyz/zinglizingli/books/core/listener/StartListener.java

@@ -66,7 +66,6 @@ public class StartListener implements ServletContextListener {
 
                         log.info("updateBooks执行中。。。。。。。。。。。。");
                         crawlSource.update();
-                        Thread.sleep(new Float(1000 * 60 * bookUpdatePeriod).longValue());
                     } catch (Exception e) {
                         log.error(e.getMessage(), e);
                     }

novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java

@@ -124,7 +124,6 @@ public class BookService {
                 bookService.insertIndexListAndContentList(newBookIndexList, newContentList);
             }
 
-            cacheUtil.del(CacheKeyConstans.NEWST_BOOK_LIST_KEY);
 
 
         }
@@ -247,7 +246,7 @@ public class BookService {
             example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(indexNum);
             List<BookContent> bookContents = bookContentMapper.selectByExample(example);
             content = bookContents.size() > 0 ? bookContents.get(0) : null;
-            cacheUtil.setObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum, content, 60 * 60 * 24);
+            cacheUtil.setObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum, content, 60 * 10);
         }
 
         return content;
@@ -510,4 +509,12 @@ public class BookService {
             }
         }
     }
+
+    /**
+     * 删除已经成功更新的解析日志
+     * */
+    public void deleteBookParseLog(Long id) {
+        bookParseLogMapper.deleteByPrimaryKey(id);
+
+    }
 }

novel-front/src/main/resources/application-crawl.yml

@@ -54,6 +54,23 @@ dingdian:
     catalog-url-pattern: <a\s+href="(/ddk\d+/all.html)">查看完整目录</a>
     catalog-pattern: <a\s+style=""\s+href="(/ddk\d+/\d+\.html)">([^/]+)</a>
 
+
+baishuzhai:
+  crawlsource:
+    index-url: https://m.baishuzhai.com
+    list-page-url: https://m.baishuzhai.com/sort/{0}/{1}.html
+    book-url-pattern: href="/(ibook/\d+/\d+)/"
+    score-pattern: <div\s+class="score">(\d+\.\d+)分</div>
+    book-name-pattern: <p class="title">([^/]+)</p>
+    author-pattern: 作者：([^/]+)<
+    status-pattern: 状态：([^/]+)</li>
+    cat-pattern: 类别：([^/]+)</li>
+    update-time-pattern: 更新：(\d+-\d+-\d+)</li>
+    pic-pattern: <img src="([^>]+)"\s+onerror="this.src=
+    intro-pattern: class="review">([^/]+)</p>
+    catalog-url-pattern: <a\s+href="(/ibook/\d+/\d+/all\.html)">查看完整目录</a>
+    catalog-pattern: <a\s+style=""\s+href="(/ibook/\d+/\d+/\d+\.html)">([^/]+)</a>
+
 biquge:
   crawlsource:
     index-url: http://m.biquge.info

novel-front/src/main/resources/application.yml

@@ -45,6 +45,19 @@ spring:
 
 
 
+
+
+  resources:
+    cache:
+      # 资源缓存时间，单位秒
+      period: 604800
+    # 开启gzip压缩
+    chain:
+      gzipped: true
+      # 启用缓存
+      cache: true
+
+
 #  mvc:
 #    static-path-pattern: /static/** #设定静态文件路径，js,css等
 mybatis:
@@ -71,10 +84,10 @@ books:
   #小说的更新间隔（分）
   updatePeriod: 1
 
-#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说  更多网站解析中，敬请期待
+#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说 ，4：百书斋 更多网站解析中，敬请期待
 crawl:
   website:
-    type: 2
+    type: 4
 
 
 

novel-front/src/main/resources/templates/books/book_content.html

@@ -19,17 +19,6 @@
     <div th:include="common/css :: css"></div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-
-
-    </script>
     <script type="text/javascript">
         function reinitIframe(){
 
@@ -477,16 +466,6 @@
 </script>
 <script>
     (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
         $("#content").css("min-height",($(window).height()-60)+"px");
 

novel-front/src/main/resources/templates/books/book_detail.html

@@ -115,30 +115,6 @@
 
     </style>
 
-    <script>
-
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function(){
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
     <script src="/js/wap_collect.js"></script>
 
 </head>

novel-front/src/main/resources/templates/books/book_index.html

@@ -39,29 +39,6 @@
     </div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function(){
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
 
 </head>
 <body>

novel-front/src/main/resources/templates/books/book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -236,18 +227,6 @@
 
 </script>
 <script>
-    (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 
 
     function toMyCollect(){

novel-front/src/main/resources/templates/books/index_v1.html

@@ -46,15 +46,6 @@
 
     </style>
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
 </head>
 <body>
@@ -247,17 +238,6 @@
 <script src="/js/wap_collect.js"></script>
 <script>
     lazyload();
-    (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        } else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 </script>
 
 

novel-front/src/main/resources/templates/books/index_v2.html

@@ -46,15 +46,6 @@
 
     </style>
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
 </head>
 <body>
@@ -293,17 +284,6 @@
 <script src="/js/wap_collect.js"></script>
 <script>
     lazyload();
-    (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        } else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 </script>
 
 

novel-front/src/main/resources/templates/books/mh_book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -247,16 +238,6 @@
 </script>
 <script>
     (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
 
         var sortCat = $("#softCat").val();

novel-front/src/main/resources/templates/books/soft_book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -272,16 +263,6 @@
 </script>
 <script>
     (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
 
         var sortCat = $("#softCat").val();

novel-front/src/main/resources/templates/index.html

@@ -50,29 +50,6 @@
     </style>
 
     <script language="javascript" type="text/javascript" src="/js/wap_collect.js"></script>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?b3a84b2ec6cc52dd088d735565b49644";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function () {
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
     <style id="tsbrowser_video_independent_player_style" type="text/css">
         [tsbrowser_force_max_size] {
             width: 100% !important;

novel-front/src/main/resources/templates/user/login.html

@@ -10,17 +10,6 @@
     <div th:include="common/css :: css"></div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-
-
-    </script>
 
 </head>
 <body id="read" >
@@ -107,18 +96,6 @@
     });
 </script>
 <script>
-    (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 
 
 

script/crawlbook/application.yml

@@ -1,14 +1,15 @@
 server: {port: 8083}
 spring:
-  datasource: {url: 'jdbc:mysql://47.106.243.172:3306/novel?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai',
+  datasource: {url: 'jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai',
-    username: novel, password: novel!8888}
+    username: root, password: test123456}
 mybatis:
   mapper-locations: classpath:mybatis/mapping/*.xml
   type-aliases-package: xyz.zinglizingli.books.po
   configuration: {log-impl: org.apache.ibatis.logging.stdout.StdOutImpl}
 mysql: {charset: utf8mb4}
-books: {lowestScore: '9.0'}
+books: {lowestScore: 9.0}
 crawl:
-  website: {type: '2'}
+  website: {type: '4'}
   soft-novel: '0'
   manhua: '0'
+logging: {config: 'classpath:logback-boot.xml'}