修复顶点小说的爬取

README.md

@@ -191,7 +191,7 @@ novel-admin ：平台后台管理系统源码（独立项目，按需安装）
 
 #### QQ交流群
 
-![mini-code](./assets/小说精品屋开源项目交流群群聊二维码.png)
+![mini-code](./assets/qq_group.png)
 
 #### 捐赠支持
 
@@ -203,6 +203,7 @@ novel-admin ：平台后台管理系统源码（独立项目，按需安装）
 
 | 捐赠者 | 金额     | 时间                        |
 | :----- | :------- | :-------------------------- |
+| **阳   | ￥10.00  | 2020年03月06日 下午14点10分 |
 | *梦    | ￥66.00  | 2019年12月27日 下午21点39分 |
 | *金名  | ￥50.00  | 2019年12月27日 下午19点29分 |
 | *天气  | ￥300.00 | 2019年12月27日 上午10点13分 |

novel-front/pom.xml

@@ -10,7 +10,7 @@
     </parent>
     <groupId>xyz.zinglizingli</groupId>
     <artifactId>novel-front</artifactId>
-    <version>2.2.0.beta</version>
+    <version>2.3.0.beta</version>
     <name>novel-front</name>
     <description>小说精品楼-前台web网站</description>
 

novel-front/src/main/java/xyz/zinglizingli/BookApplication.java

@@ -7,12 +7,14 @@ import org.springframework.boot.web.servlet.ServletComponentScan;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.context.annotation.Bean;
 import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
 
 @SpringBootApplication
 @EnableCaching
 @EnableScheduling
+@EnableAsync
 @MapperScan({"xyz.zinglizingli.*.mapper"})
 @ServletComponentScan
 public class BookApplication {

novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBaishuzhaiConfig.java

@@ -0,0 +1,29 @@
+package xyz.zinglizingli.books.core.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import xyz.zinglizingli.books.core.crawl.BaseHtmlCrawlSource;
+import xyz.zinglizingli.books.core.crawl.BiquCrawlSource;
+
+/**
+ * @author 11797
+ */
+@Slf4j
+@Configuration
+public class CrawlBaishuzhaiConfig {
+
+
+    @Bean
+    @Primary //必须加此注解，不然报错，下一个类则不需要添加
+    @ConfigurationProperties(prefix = "baishuzhai.crawlsource") // prefix值必须是application.yml中对应属性的前缀
+    @ConditionalOnProperty(prefix = "crawl.website",name = "type",havingValue = "4")
+    public BaseHtmlCrawlSource dingdianCrawlSource() {
+        return new BiquCrawlSource();
+    }
+
+
+}

novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlDingdianConfig.java

@@ -18,7 +18,6 @@ public class CrawlDingdianConfig {
 
 
     @Bean
-    @Primary //必须加此注解，不然报错，下一个类则不需要添加
     @ConfigurationProperties(prefix = "dingdian.crawlsource") // prefix值必须是application.yml中对应属性的前缀
     @ConditionalOnProperty(prefix = "crawl.website",name = "type",havingValue = "3")
     public BaseHtmlCrawlSource dingdianCrawlSource() {

novel-front/src/main/java/xyz/zinglizingli/books/core/constant/CacheKeyConstans.java

@@ -7,4 +7,5 @@ public class CacheKeyConstans {
     public static final String EMAIL_URL_PREFIX_KEY = "emailUrlPrefixKey";
     public static final String RANDOM_NEWS_CONTENT_KEY = "randomNewsContentKey";
     public static final String REC_BOOK_LIST_KEY = "recBookListKey";
+    public static final String BOOK_NUMBER_KEY= "bookNumberKey";
 }

novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java

@@ -3,15 +3,14 @@ package xyz.zinglizingli.books.core.crawl;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import xyz.zinglizingli.books.core.utils.Constants;
-import xyz.zinglizingli.books.mapper.BookParseLogMapper;
 import xyz.zinglizingli.books.po.*;
 import xyz.zinglizingli.books.service.BookService;
 import xyz.zinglizingli.books.core.utils.CatUtil;
-import xyz.zinglizingli.common.utils.ExcutorUtils;
+import xyz.zinglizingli.common.cache.CommonCacheUtil;
 import xyz.zinglizingli.common.utils.RestTemplateUtil;
 
-import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -28,18 +27,17 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
     @Autowired
     private BookService bookService;
 
+    @Autowired
+    private CommonCacheUtil cacheUtil;
+
+    @Value("${books.maxNum}")
+    private Integer maxNumBooks;
+
     @Override
     public void parse() {
 
-        Map<Integer,Date> cat2Date = bookService.queryLastUpdateTime();
+        for(int page = 1; page<= Constants.UPDATE_PAGES_ONCE; page++) {
-        Map<Integer,Date> newCat2Date = new HashMap<>();
+            String catBookListUrl = getListPageUrl().replace("{0}", "0").replace("{1}", page+"");
-        for(int i=1;i<=7;i++) {
-            Date lastUpdateTime = cat2Date.get(i);
-            Date updateTime = lastUpdateTime;
-            int page = 1;
-            do{
-                String catBookListUrl = getListPageUrl().replace("{0}", "0").replace("{1}", page + "");
-                page++;
             String forObject = RestTemplateUtil.getBodyByUtf8(catBookListUrl);
             if (forObject != null) {
                 //解析第一页书籍的数据
@@ -56,13 +54,9 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                 Matcher bookNameMatch = bookNamePatten.matcher(forObject);
 
-                    Pattern authorPatten = compile(getAuthorPattern());
-
-                    Matcher authorMatch = authorPatten.matcher(forObject);
-
                 boolean isBookNameMatch = bookNameMatch.find();
 
-                    while (isFind && scoreFind && isBookNameMatch && authorMatch.find() && updateTime.getTime()>=lastUpdateTime.getTime()) {
+                while (isFind && scoreFind && isBookNameMatch) {
 
                     try {
                         Float score = Float.parseFloat(scoreMatch.group(1));
@@ -76,30 +70,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                         String bookName = bookNameMatch.group(1);
 
-                            String author = authorMatch.group(1);
+                        bookService.addBookParseLog(bookUrl, bookName, score, (byte) 10);
-
-                            Boolean hasBook = bookService.hasBook(bookName, author);
-
-                            if (hasBook) {
-
-                                bookService.addBookParseLog(bookUrl, bookName, score);
-                            }
-
-                            String body = RestTemplateUtil.getBodyByUtf8(bookUrl);
-                            if (body != null) {
-                                Pattern updateTimePatten = compile(getUpdateTimePattern());
-                                Matcher updateTimeMatch = updateTimePatten.matcher(body);
-                                if (updateTimeMatch.find()) {
-                                    String updateTimeStr = updateTimeMatch.group(1);
-                                    SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
-                                    updateTime = format.parse(updateTimeStr);
-                                    if(!newCat2Date.containsKey(i)) {
-                                        newCat2Date.put(i, updateTime);
-                                    }
-
-
-                                }
-                            }
 
 
                     } catch (Exception e) {
@@ -116,16 +87,13 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                 }
             }
-            }while (updateTime.getTime()>=lastUpdateTime.getTime());
         }
-        bookService.updateBookUpdateTimeLog(newCat2Date);
 
     }
 
     @Override
     public void update() {
         List<BookParseLog> logs = bookService.queryBookParseLogs();
-        List<Long> successLogIds = new ArrayList<>();
         for (BookParseLog bookParseLog : logs) {
             try {
 
@@ -158,10 +126,16 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                 Pattern updateTimePatten = compile(getUpdateTimePattern());
                                 Matcher updateTimeMatch = updateTimePatten.matcher(body);
-                                if (updateTimeMatch.find()) {
+                                /*if (updateTimeMatch.find()) {
                                     String updateTimeStr = updateTimeMatch.group(1);
-                                    SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
+                                    SimpleDateFormat format ;
-                                    Date updateTime = format.parse(updateTimeStr);
+                                    if(updateTimeStr.length()>10){
+
+                                        format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
+                                    }else{
+                                        format = new SimpleDateFormat("yy-MM-dd");
+                                    }
+                                    Date updateTime = format.parse(updateTimeStr);*/
                                     Pattern picPatten = compile(getPicPattern());
                                     Matcher picMather = picPatten.matcher(body);
                                     if (picMather.find()) {
@@ -178,7 +152,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
                                         book.setScore(score > 10 ? 8.0f : score);
                                         book.setPicUrl(picSrc);
                                         book.setBookStatus(status);
-                                        book.setUpdateTime(updateTime);
+                                        book.setUpdateTime(new Date());
 
                                         List<BookIndex> indexList = new ArrayList<>();
                                         List<BookContent> contentList = new ArrayList<>();
@@ -199,8 +173,11 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                                 //查询该书籍已存在目录号
                                                 Map<Integer, BookIndex> hasIndexs = bookService.queryIndexByBookNameAndAuthor(bookName, author);
+                                                //查询数据库书籍数量
+                                                long bookNumber = bookService.queryBookNumber();
+
                                                 //更新和插入分别开，此处只做更新
-                                                if (hasIndexs.size() > 0) {
+                                                if (hasIndexs.size() > 0 || bookNumber < maxNumBooks) {
                                                     while (isFindIndex) {
                                                         BookIndex hasIndex = hasIndexs.get(indexNum);
                                                         String indexName = indexListMatch.group(2);
@@ -241,7 +218,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                                     }
                                                 }
-                                                successLogIds.add(bookParseLog.getId());
+                                                bookService.deleteBookParseLog(bookParseLog.getId());
                                             }
 
 
@@ -250,7 +227,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                     }
 
-                                }
+                                //}
                             }
                         }
 
@@ -266,7 +243,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
             }
         }
 
-        bookService.deleteBookParseLogs(successLogIds);
+
 
     }
 

novel-front/src/main/java/xyz/zinglizingli/books/core/listener/StartListener.java

@@ -66,7 +66,6 @@ public class StartListener implements ServletContextListener {
 
                         log.info("updateBooks执行中。。。。。。。。。。。。");
                         crawlSource.update();
-                        Thread.sleep(new Float(1000 * 60 * bookUpdatePeriod).longValue());
                     } catch (Exception e) {
                         log.error(e.getMessage(), e);
                     }

novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookParseLogMapper.java

@@ -27,4 +27,15 @@ public interface BookParseLogMapper {
     int updateByPrimaryKeySelective(BookParseLog record);
 
     int updateByPrimaryKey(BookParseLog record);
+
+    /**
+     * 增加小说更新次数
+     *
+     * @param logs*/
+    void addBookUpdateCount(List<BookParseLog> logs);
+
+    /**
+     * 查询解析日志
+     * */
+    List<BookParseLog> queryBookParseLogs();
 }

novel-front/src/main/java/xyz/zinglizingli/books/po/BookParseLog.java

@@ -13,6 +13,10 @@ public class BookParseLog {
 
     private Date createTime;
 
+    private Byte priority;
+
+    private Byte updateCount;
+
     public Long getId() {
         return id;
     }
@@ -52,4 +56,20 @@ public class BookParseLog {
     public void setCreateTime(Date createTime) {
         this.createTime = createTime;
     }
+
+    public Byte getPriority() {
+        return priority;
+    }
+
+    public void setPriority(Byte priority) {
+        this.priority = priority;
+    }
+
+    public Byte getUpdateCount() {
+        return updateCount;
+    }
+
+    public void setUpdateCount(Byte updateCount) {
+        this.updateCount = updateCount;
+    }
 }

novel-front/src/main/java/xyz/zinglizingli/books/po/BookParseLogExample.java

@@ -424,6 +424,126 @@ public class BookParseLogExample {
             addCriterion("create_time not between", value1, value2, "createTime");
             return (Criteria) this;
         }
+
+        public Criteria andPriorityIsNull() {
+            addCriterion("priority is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityIsNotNull() {
+            addCriterion("priority is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityEqualTo(Byte value) {
+            addCriterion("priority =", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityNotEqualTo(Byte value) {
+            addCriterion("priority <>", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityGreaterThan(Byte value) {
+            addCriterion("priority >", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityGreaterThanOrEqualTo(Byte value) {
+            addCriterion("priority >=", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityLessThan(Byte value) {
+            addCriterion("priority <", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityLessThanOrEqualTo(Byte value) {
+            addCriterion("priority <=", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityIn(List<Byte> values) {
+            addCriterion("priority in", values, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityNotIn(List<Byte> values) {
+            addCriterion("priority not in", values, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityBetween(Byte value1, Byte value2) {
+            addCriterion("priority between", value1, value2, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityNotBetween(Byte value1, Byte value2) {
+            addCriterion("priority not between", value1, value2, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountIsNull() {
+            addCriterion("update_count is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountIsNotNull() {
+            addCriterion("update_count is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountEqualTo(Byte value) {
+            addCriterion("update_count =", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountNotEqualTo(Byte value) {
+            addCriterion("update_count <>", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountGreaterThan(Byte value) {
+            addCriterion("update_count >", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountGreaterThanOrEqualTo(Byte value) {
+            addCriterion("update_count >=", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountLessThan(Byte value) {
+            addCriterion("update_count <", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountLessThanOrEqualTo(Byte value) {
+            addCriterion("update_count <=", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountIn(List<Byte> values) {
+            addCriterion("update_count in", values, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountNotIn(List<Byte> values) {
+            addCriterion("update_count not in", values, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountBetween(Byte value1, Byte value2) {
+            addCriterion("update_count between", value1, value2, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountNotBetween(Byte value1, Byte value2) {
+            addCriterion("update_count not between", value1, value2, "updateCount");
+            return (Criteria) this;
+        }
     }
 
     public static class Criteria extends GeneratedCriteria {

novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java

@@ -5,6 +5,7 @@ import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import tk.mybatis.orderbyhelper.OrderByHelper;
@@ -124,7 +125,6 @@ public class BookService {
                 bookService.insertIndexListAndContentList(newBookIndexList, newContentList);
             }
 
-            cacheUtil.del(CacheKeyConstans.NEWST_BOOK_LIST_KEY);
 
 
         }
@@ -247,7 +247,7 @@ public class BookService {
             example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(indexNum);
             List<BookContent> bookContents = bookContentMapper.selectByExample(example);
             content = bookContents.size() > 0 ? bookContents.get(0) : null;
-            cacheUtil.setObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum, content, 60 * 60 * 24);
+            cacheUtil.setObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum, content, 60 * 10);
         }
 
         return content;
@@ -440,7 +440,7 @@ public class BookService {
     /**
      * 添加解析日志
      * */
-    public void addBookParseLog(String bookUrl, String bookName, Float score) {
+    public void addBookParseLog(String bookUrl, String bookName, Float score, Byte priority) {
         BookParseLogExample example = new BookParseLogExample();
         example.createCriteria().andBookUrlEqualTo(bookUrl).andCreateTimeGreaterThan(new Date(System.currentTimeMillis()-1000*60*60));
         if(bookParseLogMapper.countByExample(example)==0) {
@@ -448,6 +448,7 @@ public class BookService {
             bookParseLog.setBookUrl(bookUrl);
             bookParseLog.setBookName(bookName);
             bookParseLog.setScore(score);
+            bookParseLog.setPriority(priority);
             bookParseLog.setCreateTime(new Date());
             bookParseLogMapper.insertSelective(bookParseLog);
         }
@@ -457,13 +458,19 @@ public class BookService {
      * 查询解析日志
      * */
     public List<BookParseLog> queryBookParseLogs() {
-        PageHelper.startPage(1,100);
+        List<BookParseLog> logs = bookParseLogMapper.queryBookParseLogs();
-        BookParseLogExample  example = new BookParseLogExample();
+        SpringUtil.getBean(BookService.class).addBookUpdateCount(logs);
-        example.setOrderByClause("create_time desc");
-        List<BookParseLog> logs = bookParseLogMapper.selectByExample(example);
         return logs;
     }
 
+    /**
+     * 增加小说更新次数
+     * */
+    @Async
+    public void addBookUpdateCount(List<BookParseLog> logs) {
+        bookParseLogMapper.addBookUpdateCount(logs);
+    }
+
     /**
      * 删除已经成功更新的解析日志
      * */
@@ -510,4 +517,25 @@ public class BookService {
             }
         }
     }
+
+    /**
+     * 删除已经成功更新的解析日志
+     * */
+    public void deleteBookParseLog(Long id) {
+        bookParseLogMapper.deleteByPrimaryKey(id);
+
+    }
+
+    /**
+     * 查询数据库书籍数量
+     * */
+    public int queryBookNumber() {
+
+        Integer bookNumber = (Integer) cacheUtil.getObject(CacheKeyConstans.BOOK_NUMBER_KEY);
+        if(bookNumber == null){
+            bookNumber = bookMapper.countByExample(new BookExample());
+            cacheUtil.setObject(CacheKeyConstans.BOOK_NUMBER_KEY,bookNumber,60*5);
+        }
+        return bookNumber;
+    }
 }

novel-front/src/main/java/xyz/zinglizingli/books/web/BookController.java

@@ -8,15 +8,9 @@ import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.ModelMap;
 import org.springframework.util.StringUtils;
-import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.*;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
 import xyz.zinglizingli.books.core.config.SeoConfig;
-import xyz.zinglizingli.books.po.Book;
+import xyz.zinglizingli.books.po.*;
-import xyz.zinglizingli.books.po.BookContent;
-import xyz.zinglizingli.books.po.BookIndex;
-import xyz.zinglizingli.books.po.ScreenBullet;
 import xyz.zinglizingli.books.service.BookService;
 import xyz.zinglizingli.books.service.UserService;
 import xyz.zinglizingli.books.vo.BookVO;
@@ -50,6 +44,27 @@ public class BookController {
     private final CommonCacheUtil commonCacheUtil;
 
 
+    /**
+     * 单本小说提交页
+     * */
+    @RequestMapping("submit.html")
+    public String bookSubmitPage(){
+        return "books/book_submit";
+    }
+
+    /**
+     * 单本小说提交
+     * */
+    @RequestMapping(method = RequestMethod.POST,value = "submit")
+    @ResponseBody
+    public Map<String, Object> bookSubmit(String bookUrl, String bookName, Float score){
+        Map<String, Object> result = new HashMap<>(2);
+        bookService.addBookParseLog(bookUrl, bookName, score, (byte) 5);
+        result.put("code", 1);
+        return result;
+    }
+
+
 
     /**
      * 精品小说搜索页
@@ -306,7 +321,7 @@ public class BookController {
     @ResponseBody
     public Map<String, Object> sendBullet(@RequestParam("contentId") Long contentId, @RequestParam("bullet") String bullet) {
         Map<String, Object> result = new HashMap<>(2);
-        bookService.sendBullet(contentId, bullet);
+        bookService.sendBullet(contentId, bullet.replaceAll("<", "&lt;").replaceAll(">", "&gt;"));
         result.put("code", 1);
         result.put("desc", "ok");
         return result;

novel-front/src/main/resources/application-crawl.yml

@@ -54,6 +54,23 @@ dingdian:
     catalog-url-pattern: <a\s+href="(/ddk\d+/all.html)">查看完整目录</a>
     catalog-pattern: <a\s+style=""\s+href="(/ddk\d+/\d+\.html)">([^/]+)</a>
 
+
+baishuzhai:
+  crawlsource:
+    index-url: https://m.baishuzhai.com
+    list-page-url: https://m.baishuzhai.com/sort/{0}/{1}.html
+    book-url-pattern: href="/(ibook/\d+/\d+)/"
+    score-pattern: <div\s+class="score">(\d+\.\d+)分</div>
+    book-name-pattern: <p class="title">([^/]+)</p>
+    author-pattern: 作者：([^/]+)<
+    status-pattern: 状态：([^/]+)</li>
+    cat-pattern: 类别：([^/]+)</li>
+    update-time-pattern: 更新：(\d+-\d+-\d+)</li>
+    pic-pattern: <img src="([^>]+)"\s+onerror="this.src=
+    intro-pattern: class="review">([^/]+)</p>
+    catalog-url-pattern: <a\s+href="(/ibook/\d+/\d+/all\.html)">查看完整目录</a>
+    catalog-pattern: <a\s+style=""\s+href="(/ibook/\d+/\d+/\d+\.html)">([^/]+)</a>
+
 biquge:
   crawlsource:
     index-url: http://m.biquge.info

novel-front/src/main/resources/application-website.yml

@@ -1,7 +1,7 @@
 #网站配置
 website:
   #网站名
-  name: 小说精品屋
+  name: 精品小说楼
   #域名
   domain: https://www.xinshumen.com
   #页面配置

novel-front/src/main/resources/application.yml

@@ -45,6 +45,19 @@ spring:
 
 
 
+
+
+  resources:
+    cache:
+      # 资源缓存时间，单位秒
+      period: 604800
+    # 开启gzip压缩
+    chain:
+      gzipped: true
+      # 启用缓存
+      cache: true
+
+
 #  mvc:
 #    static-path-pattern: /static/** #设定静态文件路径，js,css等
 mybatis:
@@ -70,11 +83,13 @@ books:
   lowestScore: 6.0
   #小说的更新间隔（分）
   updatePeriod: 1
+  #爬取小说数量
+  maxNum: 300000
 
-#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说  更多网站解析中，敬请期待
+#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说 ，4：百书斋 更多网站解析中，敬请期待
 crawl:
   website:
-    type: 2
+    type: 4
 
 
 

novel-front/src/main/resources/mybatis/generatorConfig.xml

@@ -42,7 +42,7 @@
 			<property name="enableSubPackages" value="false" />
 		</javaClientGenerator>
 
-		<table  tableName="book_update_time_log"/>
+		<table  tableName="book_parse_log"/>
 		
 		<!-- 指定数据库表 -->
 		<!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->

novel-front/src/main/resources/mybatis/mapping/BookParseLogMapper.xml

@@ -1,32 +1,35 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
-<mapper namespace="xyz.zinglizingli.books.mapper.BookParseLogMapper" >
+<mapper namespace="xyz.zinglizingli.books.mapper.BookParseLogMapper">
-  <resultMap id="BaseResultMap" type="xyz.zinglizingli.books.po.BookParseLog" >
+    <resultMap id="BaseResultMap" type="xyz.zinglizingli.books.po.BookParseLog">
-    <id column="id" property="id" jdbcType="BIGINT" />
+        <id column="id" property="id" jdbcType="BIGINT"/>
-    <result column="book_url" property="bookUrl" jdbcType="VARCHAR" />
+        <result column="book_url" property="bookUrl" jdbcType="VARCHAR"/>
-    <result column="book_name" property="bookName" jdbcType="VARCHAR" />
+        <result column="book_name" property="bookName" jdbcType="VARCHAR"/>
-    <result column="score" property="score" jdbcType="REAL" />
+        <result column="score" property="score" jdbcType="REAL"/>
-    <result column="create_time" property="createTime" jdbcType="TIMESTAMP" />
+        <result column="create_time" property="createTime" jdbcType="TIMESTAMP"/>
+        <result column="priority" property="priority" jdbcType="TINYINT"/>
+        <result column="update_count" property="updateCount" jdbcType="TINYINT"/>
     </resultMap>
-  <sql id="Example_Where_Clause" >
+    <sql id="Example_Where_Clause">
-    <where >
+        <where>
-      <foreach collection="oredCriteria" item="criteria" separator="or" >
+            <foreach collection="oredCriteria" item="criteria" separator="or">
-        <if test="criteria.valid" >
+                <if test="criteria.valid">
-          <trim prefix="(" suffix=")" prefixOverrides="and" >
+                    <trim prefix="(" suffix=")" prefixOverrides="and">
-            <foreach collection="criteria.criteria" item="criterion" >
+                        <foreach collection="criteria.criteria" item="criterion">
-              <choose >
+                            <choose>
-                <when test="criterion.noValue" >
+                                <when test="criterion.noValue">
                                     and ${criterion.condition}
                                 </when>
-                <when test="criterion.singleValue" >
+                                <when test="criterion.singleValue">
                                     and ${criterion.condition} #{criterion.value}
                                 </when>
-                <when test="criterion.betweenValue" >
+                                <when test="criterion.betweenValue">
                                     and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
                                 </when>
-                <when test="criterion.listValue" >
+                                <when test="criterion.listValue">
                                     and ${criterion.condition}
-                  <foreach collection="criterion.value" item="listItem" open="(" close=")" separator="," >
+                                    <foreach collection="criterion.value" item="listItem" open="(" close=")"
+                                             separator=",">
                                         #{listItem}
                                     </foreach>
                                 </when>
@@ -37,25 +40,26 @@
             </foreach>
         </where>
     </sql>
-  <sql id="Update_By_Example_Where_Clause" >
+    <sql id="Update_By_Example_Where_Clause">
-    <where >
+        <where>
-      <foreach collection="example.oredCriteria" item="criteria" separator="or" >
+            <foreach collection="example.oredCriteria" item="criteria" separator="or">
-        <if test="criteria.valid" >
+                <if test="criteria.valid">
-          <trim prefix="(" suffix=")" prefixOverrides="and" >
+                    <trim prefix="(" suffix=")" prefixOverrides="and">
-            <foreach collection="criteria.criteria" item="criterion" >
+                        <foreach collection="criteria.criteria" item="criterion">
-              <choose >
+                            <choose>
-                <when test="criterion.noValue" >
+                                <when test="criterion.noValue">
                                     and ${criterion.condition}
                                 </when>
-                <when test="criterion.singleValue" >
+                                <when test="criterion.singleValue">
                                     and ${criterion.condition} #{criterion.value}
                                 </when>
-                <when test="criterion.betweenValue" >
+                                <when test="criterion.betweenValue">
                                     and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
                                 </when>
-                <when test="criterion.listValue" >
+                                <when test="criterion.listValue">
                                     and ${criterion.condition}
-                  <foreach collection="criterion.value" item="listItem" open="(" close=")" separator="," >
+                                    <foreach collection="criterion.value" item="listItem" open="(" close=")"
+                                             separator=",">
                                         #{listItem}
                                     </foreach>
                                 </when>
@@ -66,146 +70,199 @@
             </foreach>
         </where>
     </sql>
-  <sql id="Base_Column_List" >
+    <sql id="Base_Column_List">
-    id, book_url, book_name, score, create_time
+    id, book_url, book_name, score, create_time, priority, update_count
   </sql>
-  <select id="selectByExample" resultMap="BaseResultMap" parameterType="xyz.zinglizingli.books.po.BookParseLogExample" >
+    <select id="selectByExample" resultMap="BaseResultMap"
+            parameterType="xyz.zinglizingli.books.po.BookParseLogExample">
         select
-    <if test="distinct" >
+        <if test="distinct">
             distinct
         </if>
-    <include refid="Base_Column_List" />
+        <include refid="Base_Column_List"/>
         from book_parse_log
-    <if test="_parameter != null" >
+        <if test="_parameter != null">
-      <include refid="Example_Where_Clause" />
+            <include refid="Example_Where_Clause"/>
         </if>
-    <if test="orderByClause != null" >
+        <if test="orderByClause != null">
             order by ${orderByClause}
         </if>
     </select>
-  <select id="selectByPrimaryKey" resultMap="BaseResultMap" parameterType="java.lang.Long" >
+    <select id="selectByPrimaryKey" resultMap="BaseResultMap" parameterType="java.lang.Long">
         select
-    <include refid="Base_Column_List" />
+        <include refid="Base_Column_List"/>
         from book_parse_log
         where id = #{id,jdbcType=BIGINT}
     </select>
-  <delete id="deleteByPrimaryKey" parameterType="java.lang.Long" >
+    <delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
     delete from book_parse_log
     where id = #{id,jdbcType=BIGINT}
   </delete>
-  <delete id="deleteByExample" parameterType="xyz.zinglizingli.books.po.BookParseLogExample" >
+    <delete id="deleteByExample" parameterType="xyz.zinglizingli.books.po.BookParseLogExample">
         delete from book_parse_log
-    <if test="_parameter != null" >
+        <if test="_parameter != null">
-      <include refid="Example_Where_Clause" />
+            <include refid="Example_Where_Clause"/>
         </if>
     </delete>
-  <insert id="insert" parameterType="xyz.zinglizingli.books.po.BookParseLog" >
+    <insert id="insert" parameterType="xyz.zinglizingli.books.po.BookParseLog">
     insert into book_parse_log (id, book_url, book_name, 
-      score, create_time)
+      score, create_time, priority, 
+      update_count)
     values (#{id,jdbcType=BIGINT}, #{bookUrl,jdbcType=VARCHAR}, #{bookName,jdbcType=VARCHAR}, 
-      #{score,jdbcType=REAL}, #{createTime,jdbcType=TIMESTAMP})
+      #{score,jdbcType=REAL}, #{createTime,jdbcType=TIMESTAMP}, #{priority,jdbcType=TINYINT}, 
+      #{updateCount,jdbcType=TINYINT})
   </insert>
-  <insert id="insertSelective" parameterType="xyz.zinglizingli.books.po.BookParseLog" >
+    <insert id="insertSelective" parameterType="xyz.zinglizingli.books.po.BookParseLog">
         insert into book_parse_log
-    <trim prefix="(" suffix=")" suffixOverrides="," >
+        <trim prefix="(" suffix=")" suffixOverrides=",">
-      <if test="id != null" >
+            <if test="id != null">
                 id,
             </if>
-      <if test="bookUrl != null" >
+            <if test="bookUrl != null">
                 book_url,
             </if>
-      <if test="bookName != null" >
+            <if test="bookName != null">
                 book_name,
             </if>
-      <if test="score != null" >
+            <if test="score != null">
                 score,
             </if>
-      <if test="createTime != null" >
+            <if test="createTime != null">
                 create_time,
             </if>
+            <if test="priority != null">
+                priority,
+            </if>
+            <if test="updateCount != null">
+                update_count,
+            </if>
         </trim>
-    <trim prefix="values (" suffix=")" suffixOverrides="," >
+        <trim prefix="values (" suffix=")" suffixOverrides=",">
-      <if test="id != null" >
+            <if test="id != null">
                 #{id,jdbcType=BIGINT},
             </if>
-      <if test="bookUrl != null" >
+            <if test="bookUrl != null">
                 #{bookUrl,jdbcType=VARCHAR},
             </if>
-      <if test="bookName != null" >
+            <if test="bookName != null">
                 #{bookName,jdbcType=VARCHAR},
             </if>
-      <if test="score != null" >
+            <if test="score != null">
                 #{score,jdbcType=REAL},
             </if>
-      <if test="createTime != null" >
+            <if test="createTime != null">
                 #{createTime,jdbcType=TIMESTAMP},
             </if>
+            <if test="priority != null">
+                #{priority,jdbcType=TINYINT},
+            </if>
+            <if test="updateCount != null">
+                #{updateCount,jdbcType=TINYINT},
+            </if>
         </trim>
     </insert>
-  <select id="countByExample" parameterType="xyz.zinglizingli.books.po.BookParseLogExample" resultType="java.lang.Integer" >
+    <select id="countByExample" parameterType="xyz.zinglizingli.books.po.BookParseLogExample"
+            resultType="java.lang.Integer">
         select count(*) from book_parse_log
-    <if test="_parameter != null" >
+        <if test="_parameter != null">
-      <include refid="Example_Where_Clause" />
+            <include refid="Example_Where_Clause"/>
         </if>
     </select>
-  <update id="updateByExampleSelective" parameterType="map" >
+    <update id="updateByExampleSelective" parameterType="map">
         update book_parse_log
-    <set >
+        <set>
-      <if test="record.id != null" >
+            <if test="record.id != null">
                 id = #{record.id,jdbcType=BIGINT},
             </if>
-      <if test="record.bookUrl != null" >
+            <if test="record.bookUrl != null">
                 book_url = #{record.bookUrl,jdbcType=VARCHAR},
             </if>
-      <if test="record.bookName != null" >
+            <if test="record.bookName != null">
                 book_name = #{record.bookName,jdbcType=VARCHAR},
             </if>
-      <if test="record.score != null" >
+            <if test="record.score != null">
                 score = #{record.score,jdbcType=REAL},
             </if>
-      <if test="record.createTime != null" >
+            <if test="record.createTime != null">
                 create_time = #{record.createTime,jdbcType=TIMESTAMP},
             </if>
+            <if test="record.priority != null">
+                priority = #{record.priority,jdbcType=TINYINT},
+            </if>
+            <if test="record.updateCount != null">
+                update_count = #{record.updateCount,jdbcType=TINYINT},
+            </if>
         </set>
-    <if test="_parameter != null" >
+        <if test="_parameter != null">
-      <include refid="Update_By_Example_Where_Clause" />
+            <include refid="Update_By_Example_Where_Clause"/>
         </if>
     </update>
-  <update id="updateByExample" parameterType="map" >
+    <update id="updateByExample" parameterType="map">
         update book_parse_log
         set id = #{record.id,jdbcType=BIGINT},
         book_url = #{record.bookUrl,jdbcType=VARCHAR},
         book_name = #{record.bookName,jdbcType=VARCHAR},
         score = #{record.score,jdbcType=REAL},
-      create_time = #{record.createTime,jdbcType=TIMESTAMP}
+        create_time = #{record.createTime,jdbcType=TIMESTAMP},
-    <if test="_parameter != null" >
+        priority = #{record.priority,jdbcType=TINYINT},
-      <include refid="Update_By_Example_Where_Clause" />
+        update_count = #{record.updateCount,jdbcType=TINYINT}
+        <if test="_parameter != null">
+            <include refid="Update_By_Example_Where_Clause"/>
         </if>
     </update>
-  <update id="updateByPrimaryKeySelective" parameterType="xyz.zinglizingli.books.po.BookParseLog" >
+    <update id="updateByPrimaryKeySelective" parameterType="xyz.zinglizingli.books.po.BookParseLog">
         update book_parse_log
-    <set >
+        <set>
-      <if test="bookUrl != null" >
+            <if test="bookUrl != null">
                 book_url = #{bookUrl,jdbcType=VARCHAR},
             </if>
-      <if test="bookName != null" >
+            <if test="bookName != null">
                 book_name = #{bookName,jdbcType=VARCHAR},
             </if>
-      <if test="score != null" >
+            <if test="score != null">
                 score = #{score,jdbcType=REAL},
             </if>
-      <if test="createTime != null" >
+            <if test="createTime != null">
                 create_time = #{createTime,jdbcType=TIMESTAMP},
             </if>
+            <if test="priority != null">
+                priority = #{priority,jdbcType=TINYINT},
+            </if>
+            <if test="updateCount != null">
+                update_count = #{updateCount,jdbcType=TINYINT},
+            </if>
         </set>
         where id = #{id,jdbcType=BIGINT}
     </update>
-  <update id="updateByPrimaryKey" parameterType="xyz.zinglizingli.books.po.BookParseLog" >
+    <update id="updateByPrimaryKey" parameterType="xyz.zinglizingli.books.po.BookParseLog">
     update book_parse_log
     set book_url = #{bookUrl,jdbcType=VARCHAR},
       book_name = #{bookName,jdbcType=VARCHAR},
       score = #{score,jdbcType=REAL},
-      create_time = #{createTime,jdbcType=TIMESTAMP}
+      create_time = #{createTime,jdbcType=TIMESTAMP},
+      priority = #{priority,jdbcType=TINYINT},
+      update_count = #{updateCount,jdbcType=TINYINT}
     where id = #{id,jdbcType=BIGINT}
   </update>
+
+    <update id="addBookUpdateCount">
+
+        update book_parse_log set update_count = update_count + 1
+        where id in
+        <foreach collection="list" item="log" separator="," open="(" close=")">
+            #{log.id}
+        </foreach>
+
+    </update>
+
+
+    <select id="queryBookParseLogs" resultMap="BaseResultMap">
+        select
+        <include refid="Base_Column_List"/>
+        from book_parse_log
+        where update_count <![CDATA[ < ]]> 5
+        order by priority asc,create_time desc
+        limit 100
+
+    </select>
 </mapper>

novel-front/src/main/resources/templates/books/book_content.html

@@ -19,17 +19,6 @@
     <div th:include="common/css :: css"></div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-
-
-    </script>
     <script type="text/javascript">
         function reinitIframe(){
 
@@ -206,7 +195,7 @@
         }
         //发送弹幕
         function sendBullet(){
-            var bullet = $("#screenBulletText").val();
+            var bullet = $("#screenBulletText").val().replace(/</g, "&lt;").replace(/>/g, "&gt;");
             var contentId = $("#contentIdHidden").val();
             if (bullet && contentId) {
                 if(bullet.length > 100){
@@ -477,16 +466,6 @@
 </script>
 <script>
     (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
         $("#content").css("min-height",($(window).height()-60)+"px");
 

novel-front/src/main/resources/templates/books/book_detail.html

@@ -115,30 +115,6 @@
 
     </style>
 
-    <script>
-
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function(){
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
     <script src="/js/wap_collect.js"></script>
 
 </head>

novel-front/src/main/resources/templates/books/book_index.html

@@ -39,29 +39,6 @@
     </div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function(){
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
 
 </head>
 <body>

novel-front/src/main/resources/templates/books/book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -236,18 +227,6 @@
 
 </script>
 <script>
-    (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 
 
     function toMyCollect(){

novel-front/src/main/resources/templates/books/book_submit.html

@@ -0,0 +1,88 @@
+<!DOCTYPE html>
+<html lang="en">
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+    <title>小说提交</title>
+
+    <div th:include="common/css :: css"></div>
+
+
+
+</head>
+<body id="read" >
+
+</div>
+
+<div style="height: 50px;line-height: 50px;text-align: center" class="layui-header header header-doc  layui-bg-cyan">
+
+    <div style="float: left;margin-left: 10px">
+        <a href="javascript:history.go(-1)">
+            <i style="font-size: 20px;color: #92B8B1;" class="layui-icon">&#xe65c;</i></a>
+    </div>
+    <b class="layui-icon">单本小说提交</b>
+    <div style="float: right;margin-right: 10px"><a href="/">
+        <i style="font-size: 20px;color: #92B8B1;" class="layui-icon">&#xe68e;</i>
+    </a>
+    </div>
+</div>
+<div style="height: 800px;padding: 20px;border: 1px solid #eee;">
+<form  class="layui-form" action="">
+    <div class="layui-form-item">
+        <label class="layui-form-label">详情页Url:</label>
+        <div class="layui-col-xs7 layui-col-sm6 layui-col-md3 layui-col-lg1" >
+            <input type="text" id="bookUrl" name="bookUrl" required  lay-verify="required" placeholder="请输入源站小说详情页Url" autocomplete="off" class="layui-input">
+        </div>
+    </div>
+    <div class="layui-form-item">
+        <label class="layui-form-label">小说名:</label>
+        <div class="layui-col-xs7 layui-col-sm6 layui-col-md3 layui-col-lg1" >
+            <input type="text" id="bookName" name="bookName" required  lay-verify="required" placeholder="请输入小说名" autocomplete="off" class="layui-input">
+        </div>
+    </div>
+    <div class="layui-form-item">
+        <label class="layui-form-label">小说评分:</label>
+        <div class="layui-col-xs7 layui-col-sm6 layui-col-md3 layui-col-lg1">
+            <input type="number" id="score" name="score" required  lay-verify="required" placeholder="请输入小说评分" autocomplete="off" class="layui-input">
+        </div>
+    </div>
+    <div class="layui-form-item">
+        <div class="layui-input-block">
+            <button class="layui-btn" lay-submit lay-filter="formDemo">提交</button>
+            <button type="reset" class="layui-btn layui-btn-primary">重置</button>
+        </div>
+    </div>
+</form>
+</div>
+
+<div th:replace="common/footer :: footer">
+</div>
+
+<a name="buttom"></a>
+</body>
+<div th:replace="common/js :: js"></div>
+
+
+<script>
+    //Demo
+    layui.use('form', function(){
+        var form = layui.form;
+
+        //监听提交
+        form.on('submit(formDemo)', function(){
+            $.post("/book/submit",{"bookUrl":$("#bookUrl").val(),"bookName":$("#bookName").val(),"score":$("#score").val()},function (data) {
+                if(data.code == 1 ){
+                    layer.alert("提交成功，任务排队中，请过一段时间后再搜索查看");
+                }else{
+                    layer.alert("提交失败");
+                }
+            });
+            return false;
+        });
+    });
+</script>
+
+
+</html>

novel-front/src/main/resources/templates/books/index.html

@@ -0,0 +1,283 @@
+<!DOCTYPE html>
+<html>
+<head>
+
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+
+    <title>精品小说楼</title>
+
+    <meta name="keywords" content="精品小说楼,精品小说,弹幕网站,弹幕,弹幕小说网站,免费小说,小说阅读,小说排行,轻小说,txt小说下载,电子书下载,动漫轻小说,日本轻小说">
+
+    <meta name="description"
+          content="精品小说楼是国内优秀的小说弹幕网站,精品小说楼提供海量热门网络小说,日本轻小说,国产轻小说,动漫小说,轻小说在线阅读和TXT小说下载,致力于网络精品小说的收集,智能计算小说评分,打造小说精品排行榜,致力于无广告无弹窗的小说阅读环境。">
+
+    <meta property="og:type" content="novel_index"/>
+    <meta property="og:title" content="精品小说楼"/>
+    <meta property="og:description"
+          content="精品小说楼是国内优秀的小说弹幕网站,精品小说楼提供海量热门网络小说,日本轻小说,国产轻小说,动漫小说,轻小说在线阅读和TXT小说下载,致力于网络精品小说的收集,智能计算小说评分,打造小说精品排行榜,致力于无广告无弹窗的小说阅读环境。"/>
+
+    <div th:include="common/css :: css"></div>
+
+    <style>
+
+        .line-limit-length {
+
+            overflow: hidden;
+
+            text-overflow: ellipsis;
+
+            white-space: nowrap;
+
+        }
+
+        .layui-nav .layui-nav-item {
+            position: relative;
+            display: inline-block;
+            vertical-align: middle;
+            line-height: 50px;
+        }
+
+        body ul.layui-nav li.layui-nav-item a {
+            display: block;
+            transition: all .3s;
+            -webkit-transition: all .3s;
+        }
+
+    </style>
+
+    <script>
+        var _hmt = _hmt || [];
+        (function () {
+            var hm = document.createElement("script");
+            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
+            var s = document.getElementsByTagName("script")[0];
+            s.parentNode.insertBefore(hm, s);
+        })();
+    </script>
+
+</head>
+<body>
+
+
+<!-- 你的HTML代码 -->
+<a name="top"></a>
+<ul class="layui-nav" lay-filter="" style="padding:0 36px;text-align: center">
+    <li class="layui-nav-item"><a href="/book/search?catId=1">玄幻小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=2">修真小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=3">都市小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=4">历史小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=6">网游小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=5">科幻小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=7">女频小说</a></li>
+    <li class="layui-nav-item"><a>完本小说</a>
+        <dl class="layui-nav-child"> <!-- 二级菜单 -->
+            <dd><a href="/book/search?bookStatus=完成">全部小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=1">玄幻小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=2">修真小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=3">都市小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=4">历史小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=6">网游小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=5">科幻小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=7">女频小说</a></dd>
+        </dl>
+    </li>
+    <li class="layui-nav-item"><a>小说排行</a>
+        <dl class="layui-nav-child"> <!-- 二级菜单 -->
+            <dd><a href="/book/search?sortBy=score">全部小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=1">玄幻小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=2">修真小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=3">都市小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=4">历史小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=6">网游小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=5">科幻小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=7">女频小说</a></dd>
+        </dl>
+    </li>
+
+</ul>
+
+<div class="layui-container">
+    <div class="layui-row">
+
+        <div class="layui-col-xs10 layui-col-sm10 layui-col-md11 layui-col-lg11" style="padding-top:1%">
+            <input id="title" type="text" name="title" required lay-verify="required" placeholder="请输入书名·作者"
+                   autocomplete="off"
+                   class="layui-input">
+        </div>
+        <div class="layui-col-xs1" style="padding: 1%">
+            <button onclick="searchBooks()" class="layui-btn" lay-submit lay-filter="formDemo">搜索</button>
+        </div>
+    </div>
+
+</div>
+<div class="layui-collapse">
+    <div class="layui-colla-item">
+        <h2 class="layui-colla-title">本站推荐</h2>
+        <div class="layui-colla-content layui-show">
+
+
+            <div class="layui-container" style="padding: 0px">
+
+                <div class="layui-row" style="text-align: center">
+                    <span th:each="recBook : ${recBooks}">
+                    <a th:href="'/book/' + ${recBook.id} + '.html'">
+                        <div style="padding: 1%" class="layui-col-xs4 layui-col-sm4 layui-col-md4 layui-col-lg4">
+                            <img style=" width:80%; height:auto; max-width:100%; max-height:100%;"
+                                 th:src="${recBook.picUrl}"/>
+                            <br/>
+                            <span th:text="${recBook.bookName}"></span>
+
+                        </div>
+                    </a>
+                        </span>
+                </div>
+            </div>
+        </div>
+    </div>
+    <div class="layui-colla-item">
+        <h2 class="layui-colla-title">热门小说推荐</h2>
+        <div class="layui-colla-content layui-show">
+
+            <div class="layui-container">
+                <div class="layui-row">
+                    <div style="margin-bottom: 5px" class="layui-col-xs12 layui-col-sm6 layui-col-md4 layui-col-lg4"
+                         th:each="hotBook : ${hotBooks}">
+                        <a th:href="'/book/' + ${hotBook.id} + '.html'">
+                            <div class="layui-col-xs5 layui-col-sm4 layui-col-md4 layui-col-lg4">
+                                <img style=" width:100px; height:125px;"
+                                     th:src="${hotBook.picUrl}"/>
+                            </div>
+                            <div class="layui-col-xs5 layui-col-sm6 layui-col-md6 layui-col-lg6">
+                                <ul>
+                                    <li class="line-limit-length" style="font-weight: bold"
+                                        th:text="${hotBook.bookName}"></li>
+                                    <li th:text="'作者：'+ ${hotBook.author}"></li>
+                                    <li style="width: 180px;height:40px;overflow: hidden"
+                                        th:text="'简介：'+ ${hotBook.bookDesc}"></li>
+                                </ul>
+                            </div>
+                            <div style="font-style: italic;color: red"
+                                 class="layui-col-xs2 layui-col-sm2 layui-col-md2 layui-col-lg2"
+                                 th:text="${hotBook.score} + '分'"></div>
+                        </a>
+                    </div>
+
+                    <!--<div th:each="hotBook : ${hotBooks}" style="margin-top: 1%"
+                     class="layui-col-xs12 layui-col-sm6 layui-col-md6 layui-col-lg6">
+                    <a th:href="'/book/'+${hotBook.id}+'.html'">
+                        <div class="layui-col-xs5 layui-col-sm3 layui-col-md3 layui-col-lg3">
+                            <img style=" width:100px; height:125px;"
+                                 th:src="${hotBook.picUrl}"/>
+                        </div>
+
+                        <div class="layui-col-xs7 layui-col-sm6 layui-col-md7 layui-col-lg7"
+                             style="float: left;padding-right: 10px">
+                            <div><b th:text="${hotBook.bookName}"></b></div>
+                            <div class="layui-col-xs8 layui-col-sm9 layui-col-md10 layui-col-lg10"
+                                 th:text="'作者：'+ ${hotBook.author}"></div>
+                            <div class="layui-col-xs3 layui-col-sm2 layui-col-md1 layui-col-lg1" style="text-align: right;">
+                            <b><i style="color: red"
+                                  th:text="${hotBook.score} + '分'"></i></b></div>
+                           </div>
+                            <div class="layui-elip layui-col-xs12 layui-col-sm9 layui-col-md10 layui-col-lg10 layui-col-sm9 layui-col-md10 layui-col-lg10"
+                                 th:text="${hotBook.bookDesc}">
+                            </div>
+
+
+
+                    </a>
+                </div>-->
+                </div>
+            </div>
+
+        </div>
+
+    </div>
+</div>
+</div>
+<div class="layui-colla-item"><a href="javascript:moreNewBooks()">
+    <h2 class="layui-colla-title">最近更新小说
+        <div style="float: right; margin-right: 20px"><i style="font-size: 14px;"
+                                                         class="layui-icon">更多&#xe65b;</i>
+        </div>
+    </h2>
+</a>
+    <div class="layui-colla-content layui-show">
+
+        <div class="layui-container" style="padding-left: 2px;padding-right: 5px">
+            <div class="layui-row">
+                <div th:each="newBook,bookStat : ${newBooks}" style="padding-bottom: 30px"
+                     class="layui-col-xs12 layui-col-sm6 layui-col-md6 layui-col-lg6">
+
+                    <a th:href="'/book/'+${newBook.id}+'.html'">
+                        <div class="line-limit-length layui-col-xs8 layui-col-sm6 layui-col-md6 layui-col-lg6"><b
+                                th:text="${bookStat.index}+1+'.'+${newBook.bookName}"></b> - <span class="layui-elip"
+                                                                                                   th:text="${newBook.author}"></span>
+                        </div>
+                        <div class="layui-col-sm3 layui-col-md3 layui-col-lg3"
+                             style="color: #FF5722;float: right;margin-right:5px"><i
+                                th:text="${#dates.format(newBook.updateTime, 'MM-dd HH:mm')}"></i></div>
+
+                        <div style="clear: both"></div>
+                        <div style="padding-left: 5px;padding-top: 5px"
+                             class="layui-elip layui-col-md11 layui-col-sm11 layui-col-lg11"
+                             th:text="'简介：'+ ${newBook.bookDesc}">
+
+                        </div>
+                    </a>
+                </div>
+
+
+            </div>
+        </div>
+
+    </div>
+</div>
+</div>
+<div th:replace="common/footer :: footer">
+</div>
+<div th:replace="common/js :: js">
+</div>
+
+<script src="/js/wap_collect.js"></script>
+<script>
+    (function () {
+        var bp = document.createElement('script');
+        var curProtocol = window.location.protocol.split(':')[0];
+        if (curProtocol === 'https') {
+            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
+        } else {
+            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
+        }
+        var s = document.getElementsByTagName("script")[0];
+        s.parentNode.insertBefore(bp, s);
+    })();
+</script>
+
+
+<script>
+
+    function moreNewBooks(event) {
+        window.location.href = "/book/search";
+    }
+
+    function searchBooks() {
+        var keywords = $("#title").val();
+        window.location.href = "/book/search?keyword=" + encodeURI(keywords);
+    }
+
+    function toMyCollect() {
+        var token = localStorage.getItem("token");
+        if (token) {
+            window.location.href = "/book/search?token=" + token;
+        } else {
+            window.location.href = "/user/login.html";
+        }
+    }
+
+</script>
+
+
+</body>
+</html>

novel-front/src/main/resources/templates/books/index_v1.html

@@ -46,15 +46,6 @@
 
     </style>
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
 </head>
 <body>
@@ -247,17 +238,6 @@
 <script src="/js/wap_collect.js"></script>
 <script>
     lazyload();
-    (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        } else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 </script>
 
 

novel-front/src/main/resources/templates/books/index_v2.html

@@ -46,15 +46,6 @@
 
     </style>
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
 </head>
 <body>
@@ -293,17 +284,6 @@
 <script src="/js/wap_collect.js"></script>
 <script>
     lazyload();
-    (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        } else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 </script>
 
 

novel-front/src/main/resources/templates/books/mh_book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -247,16 +238,6 @@
 </script>
 <script>
     (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
 
         var sortCat = $("#softCat").val();

novel-front/src/main/resources/templates/books/soft_book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -272,16 +263,6 @@
 </script>
 <script>
     (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
 
         var sortCat = $("#softCat").val();

novel-front/src/main/resources/templates/index.html

@@ -50,29 +50,6 @@
     </style>
 
     <script language="javascript" type="text/javascript" src="/js/wap_collect.js"></script>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?b3a84b2ec6cc52dd088d735565b49644";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function () {
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
     <style id="tsbrowser_video_independent_player_style" type="text/css">
         [tsbrowser_force_max_size] {
             width: 100% !important;

novel-front/src/main/resources/templates/user/login.html

@@ -10,17 +10,6 @@
     <div th:include="common/css :: css"></div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-
-
-    </script>
 
 </head>
 <body id="read" >
@@ -107,18 +96,6 @@
     });
 </script>
 <script>
-    (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 
 
 

script/crawlbook/application.yml

@@ -1,14 +1,15 @@
 server: {port: 8083}
 spring:
-  datasource: {url: 'jdbc:mysql://47.106.243.172:3306/novel?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai',
+  datasource: {url: 'jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai',
-    username: novel, password: novel!8888}
+    username: root, password: test123456}
 mybatis:
   mapper-locations: classpath:mybatis/mapping/*.xml
   type-aliases-package: xyz.zinglizingli.books.po
   configuration: {log-impl: org.apache.ibatis.logging.stdout.StdOutImpl}
 mysql: {charset: utf8mb4}
-books: {lowestScore: '9.0'}
+books: {lowestScore: 6.0}
 crawl:
-  website: {type: '2'}
+  website: {type: '4'}
-  soft-novel: '0'
+  soft-novel: 0
-  manhua: '0'
+  manhua: 0
+logging: {config: 'classpath:logback-boot.xml'}

sql/2020-04-21.sql

@@ -0,0 +1 @@
+alter table book_parse_log add column `priority` TINYINT(2) not null default 10 ;

sql/2020-04-22.sql

@@ -0,0 +1 @@
+alter table book_parse_log add column `update_count` TINYINT(2) not null default 0 ;

sql/books.sql

@@ -1,23 +1,4 @@
-/*
-Navicat MySQL Data Transfer
 
-Source Server         : aliyun_books
-Source Server Version : 80018
-Source Host           : 47.106.243.172:3306
-Source Database       : books
-
-Target Server Type    : MYSQL
-Target Server Version : 80018
-File Encoding         : 65001
-
-Date: 2019-11-15 06:10:36
-*/
-
-
-
--- ----------------------------
--- Table structure for `book`
--- ----------------------------
 DROP TABLE IF EXISTS `book`;
 CREATE TABLE `book` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
@@ -1042,3 +1023,8 @@ INSERT INTO `book_update_time_log` VALUES ('6', '6', NOW());
 INSERT INTO `book_update_time_log` VALUES ('7', '7', NOW());
 
 ALTER  TABLE  book_parse_log  drop  INDEX  uq_key_bookurl;
+
+alter table book_parse_log add column `priority` TINYINT(2) not null default 10 ;
+
+alter table book_parse_log add column `update_count` TINYINT(2) not null default 0 ;
+