修复顶点小说的爬取

README.md

@@ -191,7 +191,7 @@ novel-admin ：平台后台管理系统源码（独立项目，按需安装）
 
 #### QQ交流群
 
-![mini-code](./assets/小说精品屋开源项目交流群群聊二维码.png)
+![mini-code](./assets/qq_group.png)
 
 #### 捐赠支持
 
@@ -203,6 +203,7 @@ novel-admin ：平台后台管理系统源码（独立项目，按需安装）
 
 | 捐赠者 | 金额     | 时间                        |
 | :----- | :------- | :-------------------------- |
+| **阳   | ￥10.00  | 2020年03月06日 下午14点10分 |
 | *梦    | ￥66.00  | 2019年12月27日 下午21点39分 |
 | *金名  | ￥50.00  | 2019年12月27日 下午19点29分 |
 | *天气  | ￥300.00 | 2019年12月27日 上午10点13分 |

novel-front/src/main/java/xyz/zinglizingli/BookApplication.java

@@ -7,12 +7,14 @@ import org.springframework.boot.web.servlet.ServletComponentScan;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.context.annotation.Bean;
 import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
 
 @SpringBootApplication
 @EnableCaching
 @EnableScheduling
+@EnableAsync
 @MapperScan({"xyz.zinglizingli.*.mapper"})
 @ServletComponentScan
 public class BookApplication {

novel-front/src/main/java/xyz/zinglizingli/books/core/constant/CacheKeyConstans.java

@@ -7,4 +7,5 @@ public class CacheKeyConstans {
     public static final String EMAIL_URL_PREFIX_KEY = "emailUrlPrefixKey";
     public static final String RANDOM_NEWS_CONTENT_KEY = "randomNewsContentKey";
     public static final String REC_BOOK_LIST_KEY = "recBookListKey";
+    public static final String BOOK_NUMBER_KEY= "bookNumberKey";
 }

novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java

@@ -3,15 +3,14 @@ package xyz.zinglizingli.books.core.crawl;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import xyz.zinglizingli.books.core.utils.Constants;
-import xyz.zinglizingli.books.mapper.BookParseLogMapper;
 import xyz.zinglizingli.books.po.*;
 import xyz.zinglizingli.books.service.BookService;
 import xyz.zinglizingli.books.core.utils.CatUtil;
-import xyz.zinglizingli.common.utils.ExcutorUtils;
+import xyz.zinglizingli.common.cache.CommonCacheUtil;
 import xyz.zinglizingli.common.utils.RestTemplateUtil;
 
-import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -28,6 +27,12 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
     @Autowired
     private BookService bookService;
 
+    @Autowired
+    private CommonCacheUtil cacheUtil;
+
+    @Value("${books.maxNum}")
+    private Integer maxNumBooks;
+
     @Override
     public void parse() {
 
@@ -65,7 +70,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                         String bookName = bookNameMatch.group(1);
 
-                        bookService.addBookParseLog(bookUrl, bookName, score);
+                        bookService.addBookParseLog(bookUrl, bookName, score, (byte) 10);
 
 
                     } catch (Exception e) {
@@ -168,8 +173,11 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                                 //查询该书籍已存在目录号
                                                 Map<Integer, BookIndex> hasIndexs = bookService.queryIndexByBookNameAndAuthor(bookName, author);
+                                                //查询数据库书籍数量
+                                                long bookNumber = bookService.queryBookNumber();
+
                                                 //更新和插入分别开，此处只做更新
-                                                if (hasIndexs.size() > 0) {
+                                                if (hasIndexs.size() > 0 || bookNumber < maxNumBooks) {
                                                     while (isFindIndex) {
                                                         BookIndex hasIndex = hasIndexs.get(indexNum);
                                                         String indexName = indexListMatch.group(2);

novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookParseLogMapper.java

@@ -27,4 +27,15 @@ public interface BookParseLogMapper {
     int updateByPrimaryKeySelective(BookParseLog record);
 
     int updateByPrimaryKey(BookParseLog record);
+
+    /**
+     * 增加小说更新次数
+     *
+     * @param logs*/
+    void addBookUpdateCount(List<BookParseLog> logs);
+
+    /**
+     * 查询解析日志
+     * */
+    List<BookParseLog> queryBookParseLogs();
 }

novel-front/src/main/java/xyz/zinglizingli/books/po/BookParseLog.java

@@ -13,6 +13,10 @@ public class BookParseLog {
 
     private Date createTime;
 
+    private Byte priority;
+
+    private Byte updateCount;
+
     public Long getId() {
         return id;
     }
@@ -52,4 +56,20 @@ public class BookParseLog {
     public void setCreateTime(Date createTime) {
         this.createTime = createTime;
     }
+
+    public Byte getPriority() {
+        return priority;
+    }
+
+    public void setPriority(Byte priority) {
+        this.priority = priority;
+    }
+
+    public Byte getUpdateCount() {
+        return updateCount;
+    }
+
+    public void setUpdateCount(Byte updateCount) {
+        this.updateCount = updateCount;
+    }
 }

novel-front/src/main/java/xyz/zinglizingli/books/po/BookParseLogExample.java

@@ -424,6 +424,126 @@ public class BookParseLogExample {
             addCriterion("create_time not between", value1, value2, "createTime");
             return (Criteria) this;
         }
+
+        public Criteria andPriorityIsNull() {
+            addCriterion("priority is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityIsNotNull() {
+            addCriterion("priority is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityEqualTo(Byte value) {
+            addCriterion("priority =", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityNotEqualTo(Byte value) {
+            addCriterion("priority <>", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityGreaterThan(Byte value) {
+            addCriterion("priority >", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityGreaterThanOrEqualTo(Byte value) {
+            addCriterion("priority >=", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityLessThan(Byte value) {
+            addCriterion("priority <", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityLessThanOrEqualTo(Byte value) {
+            addCriterion("priority <=", value, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityIn(List<Byte> values) {
+            addCriterion("priority in", values, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityNotIn(List<Byte> values) {
+            addCriterion("priority not in", values, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityBetween(Byte value1, Byte value2) {
+            addCriterion("priority between", value1, value2, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andPriorityNotBetween(Byte value1, Byte value2) {
+            addCriterion("priority not between", value1, value2, "priority");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountIsNull() {
+            addCriterion("update_count is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountIsNotNull() {
+            addCriterion("update_count is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountEqualTo(Byte value) {
+            addCriterion("update_count =", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountNotEqualTo(Byte value) {
+            addCriterion("update_count <>", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountGreaterThan(Byte value) {
+            addCriterion("update_count >", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountGreaterThanOrEqualTo(Byte value) {
+            addCriterion("update_count >=", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountLessThan(Byte value) {
+            addCriterion("update_count <", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountLessThanOrEqualTo(Byte value) {
+            addCriterion("update_count <=", value, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountIn(List<Byte> values) {
+            addCriterion("update_count in", values, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountNotIn(List<Byte> values) {
+            addCriterion("update_count not in", values, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountBetween(Byte value1, Byte value2) {
+            addCriterion("update_count between", value1, value2, "updateCount");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateCountNotBetween(Byte value1, Byte value2) {
+            addCriterion("update_count not between", value1, value2, "updateCount");
+            return (Criteria) this;
+        }
     }
 
     public static class Criteria extends GeneratedCriteria {

novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java

@@ -5,6 +5,7 @@ import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import tk.mybatis.orderbyhelper.OrderByHelper;
@@ -439,7 +440,7 @@ public class BookService {
     /**
      * 添加解析日志
      * */
-    public void addBookParseLog(String bookUrl, String bookName, Float score) {
+    public void addBookParseLog(String bookUrl, String bookName, Float score, Byte priority) {
         BookParseLogExample example = new BookParseLogExample();
         example.createCriteria().andBookUrlEqualTo(bookUrl).andCreateTimeGreaterThan(new Date(System.currentTimeMillis()-1000*60*60));
         if(bookParseLogMapper.countByExample(example)==0) {
@@ -447,6 +448,7 @@ public class BookService {
             bookParseLog.setBookUrl(bookUrl);
             bookParseLog.setBookName(bookName);
             bookParseLog.setScore(score);
+            bookParseLog.setPriority(priority);
             bookParseLog.setCreateTime(new Date());
             bookParseLogMapper.insertSelective(bookParseLog);
         }
@@ -456,13 +458,19 @@ public class BookService {
      * 查询解析日志
      * */
     public List<BookParseLog> queryBookParseLogs() {
-        PageHelper.startPage(1,100);
-        BookParseLogExample  example = new BookParseLogExample();
-        example.setOrderByClause("create_time desc");
-        List<BookParseLog> logs = bookParseLogMapper.selectByExample(example);
+        List<BookParseLog> logs = bookParseLogMapper.queryBookParseLogs();
+        SpringUtil.getBean(BookService.class).addBookUpdateCount(logs);
         return logs;
     }
 
+    /**
+     * 增加小说更新次数
+     * */
+    @Async
+    public void addBookUpdateCount(List<BookParseLog> logs) {
+        bookParseLogMapper.addBookUpdateCount(logs);
+    }
+
     /**
      * 删除已经成功更新的解析日志
      * */
@@ -517,4 +525,17 @@ public class BookService {
         bookParseLogMapper.deleteByPrimaryKey(id);
 
     }
+
+    /**
+     * 查询数据库书籍数量
+     * */
+    public int queryBookNumber() {
+
+        Integer bookNumber = (Integer) cacheUtil.getObject(CacheKeyConstans.BOOK_NUMBER_KEY);
+        if(bookNumber == null){
+            bookNumber = bookMapper.countByExample(new BookExample());
+            cacheUtil.setObject(CacheKeyConstans.BOOK_NUMBER_KEY,bookNumber,60*5);
+        }
+        return bookNumber;
+    }
 }

novel-front/src/main/java/xyz/zinglizingli/books/web/BookController.java

@@ -8,15 +8,9 @@ import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.ModelMap;
 import org.springframework.util.StringUtils;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
 import xyz.zinglizingli.books.core.config.SeoConfig;
-import xyz.zinglizingli.books.po.Book;
-import xyz.zinglizingli.books.po.BookContent;
-import xyz.zinglizingli.books.po.BookIndex;
-import xyz.zinglizingli.books.po.ScreenBullet;
+import xyz.zinglizingli.books.po.*;
 import xyz.zinglizingli.books.service.BookService;
 import xyz.zinglizingli.books.service.UserService;
 import xyz.zinglizingli.books.vo.BookVO;
@@ -50,6 +44,27 @@ public class BookController {
     private final CommonCacheUtil commonCacheUtil;
 
 
+    /**
+     * 单本小说提交页
+     * */
+    @RequestMapping("submit.html")
+    public String bookSubmitPage(){
+        return "books/book_submit";
+    }
+
+    /**
+     * 单本小说提交
+     * */
+    @RequestMapping(method = RequestMethod.POST,value = "submit")
+    @ResponseBody
+    public Map<String, Object> bookSubmit(String bookUrl, String bookName, Float score){
+        Map<String, Object> result = new HashMap<>(2);
+        bookService.addBookParseLog(bookUrl, bookName, score, (byte) 5);
+        result.put("code", 1);
+        return result;
+    }
+
+
 
     /**
      * 精品小说搜索页
@@ -306,7 +321,7 @@ public class BookController {
     @ResponseBody
     public Map<String, Object> sendBullet(@RequestParam("contentId") Long contentId, @RequestParam("bullet") String bullet) {
         Map<String, Object> result = new HashMap<>(2);
-        bookService.sendBullet(contentId, bullet);
+        bookService.sendBullet(contentId, bullet.replaceAll("<", "&lt;").replaceAll(">", "&gt;"));
         result.put("code", 1);
         result.put("desc", "ok");
         return result;

novel-front/src/main/resources/application-website.yml

@@ -1,7 +1,7 @@
 #网站配置
 website:
   #网站名
-  name: 小说精品屋
+  name: 精品小说楼
   #域名
   domain: https://www.xinshumen.com
   #页面配置

novel-front/src/main/resources/application.yml

@@ -83,6 +83,8 @@ books:
   lowestScore: 6.0
   #小说的更新间隔（分）
   updatePeriod: 1
+  #爬取小说数量
+  maxNum: 300000
 
 #爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说 ，4：百书斋 更多网站解析中，敬请期待
 crawl:

novel-front/src/main/resources/mybatis/generatorConfig.xml

@@ -42,7 +42,7 @@
 			<property name="enableSubPackages" value="false" />
 		</javaClientGenerator>
 
-		<table  tableName="book_update_time_log"/>
+		<table  tableName="book_parse_log"/>
 		
 		<!-- 指定数据库表 -->
 		<!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->

novel-front/src/main/resources/mybatis/mapping/BookParseLogMapper.xml

@@ -1,32 +1,35 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
-<mapper namespace="xyz.zinglizingli.books.mapper.BookParseLogMapper" >
-  <resultMap id="BaseResultMap" type="xyz.zinglizingli.books.po.BookParseLog" >
-    <id column="id" property="id" jdbcType="BIGINT" />
-    <result column="book_url" property="bookUrl" jdbcType="VARCHAR" />
-    <result column="book_name" property="bookName" jdbcType="VARCHAR" />
-    <result column="score" property="score" jdbcType="REAL" />
-    <result column="create_time" property="createTime" jdbcType="TIMESTAMP" />
+<mapper namespace="xyz.zinglizingli.books.mapper.BookParseLogMapper">
+    <resultMap id="BaseResultMap" type="xyz.zinglizingli.books.po.BookParseLog">
+        <id column="id" property="id" jdbcType="BIGINT"/>
+        <result column="book_url" property="bookUrl" jdbcType="VARCHAR"/>
+        <result column="book_name" property="bookName" jdbcType="VARCHAR"/>
+        <result column="score" property="score" jdbcType="REAL"/>
+        <result column="create_time" property="createTime" jdbcType="TIMESTAMP"/>
+        <result column="priority" property="priority" jdbcType="TINYINT"/>
+        <result column="update_count" property="updateCount" jdbcType="TINYINT"/>
     </resultMap>
-  <sql id="Example_Where_Clause" >
-    <where >
-      <foreach collection="oredCriteria" item="criteria" separator="or" >
-        <if test="criteria.valid" >
-          <trim prefix="(" suffix=")" prefixOverrides="and" >
-            <foreach collection="criteria.criteria" item="criterion" >
-              <choose >
-                <when test="criterion.noValue" >
+    <sql id="Example_Where_Clause">
+        <where>
+            <foreach collection="oredCriteria" item="criteria" separator="or">
+                <if test="criteria.valid">
+                    <trim prefix="(" suffix=")" prefixOverrides="and">
+                        <foreach collection="criteria.criteria" item="criterion">
+                            <choose>
+                                <when test="criterion.noValue">
                                     and ${criterion.condition}
                                 </when>
-                <when test="criterion.singleValue" >
+                                <when test="criterion.singleValue">
                                     and ${criterion.condition} #{criterion.value}
                                 </when>
-                <when test="criterion.betweenValue" >
+                                <when test="criterion.betweenValue">
                                     and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
                                 </when>
-                <when test="criterion.listValue" >
+                                <when test="criterion.listValue">
                                     and ${criterion.condition}
-                  <foreach collection="criterion.value" item="listItem" open="(" close=")" separator="," >
+                                    <foreach collection="criterion.value" item="listItem" open="(" close=")"
+                                             separator=",">
                                         #{listItem}
                                     </foreach>
                                 </when>
@@ -37,25 +40,26 @@
             </foreach>
         </where>
     </sql>
-  <sql id="Update_By_Example_Where_Clause" >
-    <where >
-      <foreach collection="example.oredCriteria" item="criteria" separator="or" >
-        <if test="criteria.valid" >
-          <trim prefix="(" suffix=")" prefixOverrides="and" >
-            <foreach collection="criteria.criteria" item="criterion" >
-              <choose >
-                <when test="criterion.noValue" >
+    <sql id="Update_By_Example_Where_Clause">
+        <where>
+            <foreach collection="example.oredCriteria" item="criteria" separator="or">
+                <if test="criteria.valid">
+                    <trim prefix="(" suffix=")" prefixOverrides="and">
+                        <foreach collection="criteria.criteria" item="criterion">
+                            <choose>
+                                <when test="criterion.noValue">
                                     and ${criterion.condition}
                                 </when>
-                <when test="criterion.singleValue" >
+                                <when test="criterion.singleValue">
                                     and ${criterion.condition} #{criterion.value}
                                 </when>
-                <when test="criterion.betweenValue" >
+                                <when test="criterion.betweenValue">
                                     and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
                                 </when>
-                <when test="criterion.listValue" >
+                                <when test="criterion.listValue">
                                     and ${criterion.condition}
-                  <foreach collection="criterion.value" item="listItem" open="(" close=")" separator="," >
+                                    <foreach collection="criterion.value" item="listItem" open="(" close=")"
+                                             separator=",">
                                         #{listItem}
                                     </foreach>
                                 </when>
@@ -66,146 +70,199 @@
             </foreach>
         </where>
     </sql>
-  <sql id="Base_Column_List" >
-    id, book_url, book_name, score, create_time
+    <sql id="Base_Column_List">
+    id, book_url, book_name, score, create_time, priority, update_count
   </sql>
-  <select id="selectByExample" resultMap="BaseResultMap" parameterType="xyz.zinglizingli.books.po.BookParseLogExample" >
+    <select id="selectByExample" resultMap="BaseResultMap"
+            parameterType="xyz.zinglizingli.books.po.BookParseLogExample">
         select
-    <if test="distinct" >
+        <if test="distinct">
             distinct
         </if>
-    <include refid="Base_Column_List" />
+        <include refid="Base_Column_List"/>
         from book_parse_log
-    <if test="_parameter != null" >
-      <include refid="Example_Where_Clause" />
+        <if test="_parameter != null">
+            <include refid="Example_Where_Clause"/>
         </if>
-    <if test="orderByClause != null" >
+        <if test="orderByClause != null">
             order by ${orderByClause}
         </if>
     </select>
-  <select id="selectByPrimaryKey" resultMap="BaseResultMap" parameterType="java.lang.Long" >
+    <select id="selectByPrimaryKey" resultMap="BaseResultMap" parameterType="java.lang.Long">
         select
-    <include refid="Base_Column_List" />
+        <include refid="Base_Column_List"/>
         from book_parse_log
         where id = #{id,jdbcType=BIGINT}
     </select>
-  <delete id="deleteByPrimaryKey" parameterType="java.lang.Long" >
+    <delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
     delete from book_parse_log
     where id = #{id,jdbcType=BIGINT}
   </delete>
-  <delete id="deleteByExample" parameterType="xyz.zinglizingli.books.po.BookParseLogExample" >
+    <delete id="deleteByExample" parameterType="xyz.zinglizingli.books.po.BookParseLogExample">
         delete from book_parse_log
-    <if test="_parameter != null" >
-      <include refid="Example_Where_Clause" />
+        <if test="_parameter != null">
+            <include refid="Example_Where_Clause"/>
         </if>
     </delete>
-  <insert id="insert" parameterType="xyz.zinglizingli.books.po.BookParseLog" >
+    <insert id="insert" parameterType="xyz.zinglizingli.books.po.BookParseLog">
     insert into book_parse_log (id, book_url, book_name, 
-      score, create_time)
+      score, create_time, priority, 
+      update_count)
     values (#{id,jdbcType=BIGINT}, #{bookUrl,jdbcType=VARCHAR}, #{bookName,jdbcType=VARCHAR}, 
-      #{score,jdbcType=REAL}, #{createTime,jdbcType=TIMESTAMP})
+      #{score,jdbcType=REAL}, #{createTime,jdbcType=TIMESTAMP}, #{priority,jdbcType=TINYINT}, 
+      #{updateCount,jdbcType=TINYINT})
   </insert>
-  <insert id="insertSelective" parameterType="xyz.zinglizingli.books.po.BookParseLog" >
+    <insert id="insertSelective" parameterType="xyz.zinglizingli.books.po.BookParseLog">
         insert into book_parse_log
-    <trim prefix="(" suffix=")" suffixOverrides="," >
-      <if test="id != null" >
+        <trim prefix="(" suffix=")" suffixOverrides=",">
+            <if test="id != null">
                 id,
             </if>
-      <if test="bookUrl != null" >
+            <if test="bookUrl != null">
                 book_url,
             </if>
-      <if test="bookName != null" >
+            <if test="bookName != null">
                 book_name,
             </if>
-      <if test="score != null" >
+            <if test="score != null">
                 score,
             </if>
-      <if test="createTime != null" >
+            <if test="createTime != null">
                 create_time,
             </if>
+            <if test="priority != null">
+                priority,
+            </if>
+            <if test="updateCount != null">
+                update_count,
+            </if>
         </trim>
-    <trim prefix="values (" suffix=")" suffixOverrides="," >
-      <if test="id != null" >
+        <trim prefix="values (" suffix=")" suffixOverrides=",">
+            <if test="id != null">
                 #{id,jdbcType=BIGINT},
             </if>
-      <if test="bookUrl != null" >
+            <if test="bookUrl != null">
                 #{bookUrl,jdbcType=VARCHAR},
             </if>
-      <if test="bookName != null" >
+            <if test="bookName != null">
                 #{bookName,jdbcType=VARCHAR},
             </if>
-      <if test="score != null" >
+            <if test="score != null">
                 #{score,jdbcType=REAL},
             </if>
-      <if test="createTime != null" >
+            <if test="createTime != null">
                 #{createTime,jdbcType=TIMESTAMP},
             </if>
+            <if test="priority != null">
+                #{priority,jdbcType=TINYINT},
+            </if>
+            <if test="updateCount != null">
+                #{updateCount,jdbcType=TINYINT},
+            </if>
         </trim>
     </insert>
-  <select id="countByExample" parameterType="xyz.zinglizingli.books.po.BookParseLogExample" resultType="java.lang.Integer" >
+    <select id="countByExample" parameterType="xyz.zinglizingli.books.po.BookParseLogExample"
+            resultType="java.lang.Integer">
         select count(*) from book_parse_log
-    <if test="_parameter != null" >
-      <include refid="Example_Where_Clause" />
+        <if test="_parameter != null">
+            <include refid="Example_Where_Clause"/>
         </if>
     </select>
-  <update id="updateByExampleSelective" parameterType="map" >
+    <update id="updateByExampleSelective" parameterType="map">
         update book_parse_log
-    <set >
-      <if test="record.id != null" >
+        <set>
+            <if test="record.id != null">
                 id = #{record.id,jdbcType=BIGINT},
             </if>
-      <if test="record.bookUrl != null" >
+            <if test="record.bookUrl != null">
                 book_url = #{record.bookUrl,jdbcType=VARCHAR},
             </if>
-      <if test="record.bookName != null" >
+            <if test="record.bookName != null">
                 book_name = #{record.bookName,jdbcType=VARCHAR},
             </if>
-      <if test="record.score != null" >
+            <if test="record.score != null">
                 score = #{record.score,jdbcType=REAL},
             </if>
-      <if test="record.createTime != null" >
+            <if test="record.createTime != null">
                 create_time = #{record.createTime,jdbcType=TIMESTAMP},
             </if>
+            <if test="record.priority != null">
+                priority = #{record.priority,jdbcType=TINYINT},
+            </if>
+            <if test="record.updateCount != null">
+                update_count = #{record.updateCount,jdbcType=TINYINT},
+            </if>
         </set>
-    <if test="_parameter != null" >
-      <include refid="Update_By_Example_Where_Clause" />
+        <if test="_parameter != null">
+            <include refid="Update_By_Example_Where_Clause"/>
         </if>
     </update>
-  <update id="updateByExample" parameterType="map" >
+    <update id="updateByExample" parameterType="map">
         update book_parse_log
         set id = #{record.id,jdbcType=BIGINT},
         book_url = #{record.bookUrl,jdbcType=VARCHAR},
         book_name = #{record.bookName,jdbcType=VARCHAR},
         score = #{record.score,jdbcType=REAL},
-      create_time = #{record.createTime,jdbcType=TIMESTAMP}
-    <if test="_parameter != null" >
-      <include refid="Update_By_Example_Where_Clause" />
+        create_time = #{record.createTime,jdbcType=TIMESTAMP},
+        priority = #{record.priority,jdbcType=TINYINT},
+        update_count = #{record.updateCount,jdbcType=TINYINT}
+        <if test="_parameter != null">
+            <include refid="Update_By_Example_Where_Clause"/>
         </if>
     </update>
-  <update id="updateByPrimaryKeySelective" parameterType="xyz.zinglizingli.books.po.BookParseLog" >
+    <update id="updateByPrimaryKeySelective" parameterType="xyz.zinglizingli.books.po.BookParseLog">
         update book_parse_log
-    <set >
-      <if test="bookUrl != null" >
+        <set>
+            <if test="bookUrl != null">
                 book_url = #{bookUrl,jdbcType=VARCHAR},
             </if>
-      <if test="bookName != null" >
+            <if test="bookName != null">
                 book_name = #{bookName,jdbcType=VARCHAR},
             </if>
-      <if test="score != null" >
+            <if test="score != null">
                 score = #{score,jdbcType=REAL},
             </if>
-      <if test="createTime != null" >
+            <if test="createTime != null">
                 create_time = #{createTime,jdbcType=TIMESTAMP},
             </if>
+            <if test="priority != null">
+                priority = #{priority,jdbcType=TINYINT},
+            </if>
+            <if test="updateCount != null">
+                update_count = #{updateCount,jdbcType=TINYINT},
+            </if>
         </set>
         where id = #{id,jdbcType=BIGINT}
     </update>
-  <update id="updateByPrimaryKey" parameterType="xyz.zinglizingli.books.po.BookParseLog" >
+    <update id="updateByPrimaryKey" parameterType="xyz.zinglizingli.books.po.BookParseLog">
     update book_parse_log
     set book_url = #{bookUrl,jdbcType=VARCHAR},
       book_name = #{bookName,jdbcType=VARCHAR},
       score = #{score,jdbcType=REAL},
-      create_time = #{createTime,jdbcType=TIMESTAMP}
+      create_time = #{createTime,jdbcType=TIMESTAMP},
+      priority = #{priority,jdbcType=TINYINT},
+      update_count = #{updateCount,jdbcType=TINYINT}
     where id = #{id,jdbcType=BIGINT}
   </update>
+
+    <update id="addBookUpdateCount">
+
+        update book_parse_log set update_count = update_count + 1
+        where id in
+        <foreach collection="list" item="log" separator="," open="(" close=")">
+            #{log.id}
+        </foreach>
+
+    </update>
+
+
+    <select id="queryBookParseLogs" resultMap="BaseResultMap">
+        select
+        <include refid="Base_Column_List"/>
+        from book_parse_log
+        where update_count <![CDATA[ < ]]> 5
+        order by priority asc,create_time desc
+        limit 100
+
+    </select>
 </mapper>

novel-front/src/main/resources/templates/books/book_content.html

@@ -195,7 +195,7 @@
         }
         //发送弹幕
         function sendBullet(){
-            var bullet = $("#screenBulletText").val();
+            var bullet = $("#screenBulletText").val().replace(/</g, "&lt;").replace(/>/g, "&gt;");
             var contentId = $("#contentIdHidden").val();
             if (bullet && contentId) {
                 if(bullet.length > 100){

novel-front/src/main/resources/templates/books/book_submit.html

@@ -0,0 +1,88 @@
+<!DOCTYPE html>
+<html lang="en">
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+    <title>小说提交</title>
+
+    <div th:include="common/css :: css"></div>
+
+
+
+</head>
+<body id="read" >
+
+</div>
+
+<div style="height: 50px;line-height: 50px;text-align: center" class="layui-header header header-doc  layui-bg-cyan">
+
+    <div style="float: left;margin-left: 10px">
+        <a href="javascript:history.go(-1)">
+            <i style="font-size: 20px;color: #92B8B1;" class="layui-icon">&#xe65c;</i></a>
+    </div>
+    <b class="layui-icon">单本小说提交</b>
+    <div style="float: right;margin-right: 10px"><a href="/">
+        <i style="font-size: 20px;color: #92B8B1;" class="layui-icon">&#xe68e;</i>
+    </a>
+    </div>
+</div>
+<div style="height: 800px;padding: 20px;border: 1px solid #eee;">
+<form  class="layui-form" action="">
+    <div class="layui-form-item">
+        <label class="layui-form-label">详情页Url:</label>
+        <div class="layui-col-xs7 layui-col-sm6 layui-col-md3 layui-col-lg1" >
+            <input type="text" id="bookUrl" name="bookUrl" required  lay-verify="required" placeholder="请输入源站小说详情页Url" autocomplete="off" class="layui-input">
+        </div>
+    </div>
+    <div class="layui-form-item">
+        <label class="layui-form-label">小说名:</label>
+        <div class="layui-col-xs7 layui-col-sm6 layui-col-md3 layui-col-lg1" >
+            <input type="text" id="bookName" name="bookName" required  lay-verify="required" placeholder="请输入小说名" autocomplete="off" class="layui-input">
+        </div>
+    </div>
+    <div class="layui-form-item">
+        <label class="layui-form-label">小说评分:</label>
+        <div class="layui-col-xs7 layui-col-sm6 layui-col-md3 layui-col-lg1">
+            <input type="number" id="score" name="score" required  lay-verify="required" placeholder="请输入小说评分" autocomplete="off" class="layui-input">
+        </div>
+    </div>
+    <div class="layui-form-item">
+        <div class="layui-input-block">
+            <button class="layui-btn" lay-submit lay-filter="formDemo">提交</button>
+            <button type="reset" class="layui-btn layui-btn-primary">重置</button>
+        </div>
+    </div>
+</form>
+</div>
+
+<div th:replace="common/footer :: footer">
+</div>
+
+<a name="buttom"></a>
+</body>
+<div th:replace="common/js :: js"></div>
+
+
+<script>
+    //Demo
+    layui.use('form', function(){
+        var form = layui.form;
+
+        //监听提交
+        form.on('submit(formDemo)', function(){
+            $.post("/book/submit",{"bookUrl":$("#bookUrl").val(),"bookName":$("#bookName").val(),"score":$("#score").val()},function (data) {
+                if(data.code == 1 ){
+                    layer.alert("提交成功，任务排队中，请过一段时间后再搜索查看");
+                }else{
+                    layer.alert("提交失败");
+                }
+            });
+            return false;
+        });
+    });
+</script>
+
+
+</html>

novel-front/src/main/resources/templates/books/index.html

@@ -0,0 +1,283 @@
+<!DOCTYPE html>
+<html>
+<head>
+
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+
+    <title>精品小说楼</title>
+
+    <meta name="keywords" content="精品小说楼,精品小说,弹幕网站,弹幕,弹幕小说网站,免费小说,小说阅读,小说排行,轻小说,txt小说下载,电子书下载,动漫轻小说,日本轻小说">
+
+    <meta name="description"
+          content="精品小说楼是国内优秀的小说弹幕网站,精品小说楼提供海量热门网络小说,日本轻小说,国产轻小说,动漫小说,轻小说在线阅读和TXT小说下载,致力于网络精品小说的收集,智能计算小说评分,打造小说精品排行榜,致力于无广告无弹窗的小说阅读环境。">
+
+    <meta property="og:type" content="novel_index"/>
+    <meta property="og:title" content="精品小说楼"/>
+    <meta property="og:description"
+          content="精品小说楼是国内优秀的小说弹幕网站,精品小说楼提供海量热门网络小说,日本轻小说,国产轻小说,动漫小说,轻小说在线阅读和TXT小说下载,致力于网络精品小说的收集,智能计算小说评分,打造小说精品排行榜,致力于无广告无弹窗的小说阅读环境。"/>
+
+    <div th:include="common/css :: css"></div>
+
+    <style>
+
+        .line-limit-length {
+
+            overflow: hidden;
+
+            text-overflow: ellipsis;
+
+            white-space: nowrap;
+
+        }
+
+        .layui-nav .layui-nav-item {
+            position: relative;
+            display: inline-block;
+            vertical-align: middle;
+            line-height: 50px;
+        }
+
+        body ul.layui-nav li.layui-nav-item a {
+            display: block;
+            transition: all .3s;
+            -webkit-transition: all .3s;
+        }
+
+    </style>
+
+    <script>
+        var _hmt = _hmt || [];
+        (function () {
+            var hm = document.createElement("script");
+            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
+            var s = document.getElementsByTagName("script")[0];
+            s.parentNode.insertBefore(hm, s);
+        })();
+    </script>
+
+</head>
+<body>
+
+
+<!-- 你的HTML代码 -->
+<a name="top"></a>
+<ul class="layui-nav" lay-filter="" style="padding:0 36px;text-align: center">
+    <li class="layui-nav-item"><a href="/book/search?catId=1">玄幻小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=2">修真小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=3">都市小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=4">历史小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=6">网游小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=5">科幻小说</a></li>
+    <li class="layui-nav-item"><a href="/book/search?catId=7">女频小说</a></li>
+    <li class="layui-nav-item"><a>完本小说</a>
+        <dl class="layui-nav-child"> <!-- 二级菜单 -->
+            <dd><a href="/book/search?bookStatus=完成">全部小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=1">玄幻小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=2">修真小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=3">都市小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=4">历史小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=6">网游小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=5">科幻小说</a></dd>
+            <dd><a href="/book/search?bookStatus=完成&catId=7">女频小说</a></dd>
+        </dl>
+    </li>
+    <li class="layui-nav-item"><a>小说排行</a>
+        <dl class="layui-nav-child"> <!-- 二级菜单 -->
+            <dd><a href="/book/search?sortBy=score">全部小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=1">玄幻小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=2">修真小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=3">都市小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=4">历史小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=6">网游小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=5">科幻小说</a></dd>
+            <dd><a href="/book/search?sortBy=score&catId=7">女频小说</a></dd>
+        </dl>
+    </li>
+
+</ul>
+
+<div class="layui-container">
+    <div class="layui-row">
+
+        <div class="layui-col-xs10 layui-col-sm10 layui-col-md11 layui-col-lg11" style="padding-top:1%">
+            <input id="title" type="text" name="title" required lay-verify="required" placeholder="请输入书名·作者"
+                   autocomplete="off"
+                   class="layui-input">
+        </div>
+        <div class="layui-col-xs1" style="padding: 1%">
+            <button onclick="searchBooks()" class="layui-btn" lay-submit lay-filter="formDemo">搜索</button>
+        </div>
+    </div>
+
+</div>
+<div class="layui-collapse">
+    <div class="layui-colla-item">
+        <h2 class="layui-colla-title">本站推荐</h2>
+        <div class="layui-colla-content layui-show">
+
+
+            <div class="layui-container" style="padding: 0px">
+
+                <div class="layui-row" style="text-align: center">
+                    <span th:each="recBook : ${recBooks}">
+                    <a th:href="'/book/' + ${recBook.id} + '.html'">
+                        <div style="padding: 1%" class="layui-col-xs4 layui-col-sm4 layui-col-md4 layui-col-lg4">
+                            <img style=" width:80%; height:auto; max-width:100%; max-height:100%;"
+                                 th:src="${recBook.picUrl}"/>
+                            <br/>
+                            <span th:text="${recBook.bookName}"></span>
+
+                        </div>
+                    </a>
+                        </span>
+                </div>
+            </div>
+        </div>
+    </div>
+    <div class="layui-colla-item">
+        <h2 class="layui-colla-title">热门小说推荐</h2>
+        <div class="layui-colla-content layui-show">
+
+            <div class="layui-container">
+                <div class="layui-row">
+                    <div style="margin-bottom: 5px" class="layui-col-xs12 layui-col-sm6 layui-col-md4 layui-col-lg4"
+                         th:each="hotBook : ${hotBooks}">
+                        <a th:href="'/book/' + ${hotBook.id} + '.html'">
+                            <div class="layui-col-xs5 layui-col-sm4 layui-col-md4 layui-col-lg4">
+                                <img style=" width:100px; height:125px;"
+                                     th:src="${hotBook.picUrl}"/>
+                            </div>
+                            <div class="layui-col-xs5 layui-col-sm6 layui-col-md6 layui-col-lg6">
+                                <ul>
+                                    <li class="line-limit-length" style="font-weight: bold"
+                                        th:text="${hotBook.bookName}"></li>
+                                    <li th:text="'作者：'+ ${hotBook.author}"></li>
+                                    <li style="width: 180px;height:40px;overflow: hidden"
+                                        th:text="'简介：'+ ${hotBook.bookDesc}"></li>
+                                </ul>
+                            </div>
+                            <div style="font-style: italic;color: red"
+                                 class="layui-col-xs2 layui-col-sm2 layui-col-md2 layui-col-lg2"
+                                 th:text="${hotBook.score} + '分'"></div>
+                        </a>
+                    </div>
+
+                    <!--<div th:each="hotBook : ${hotBooks}" style="margin-top: 1%"
+                     class="layui-col-xs12 layui-col-sm6 layui-col-md6 layui-col-lg6">
+                    <a th:href="'/book/'+${hotBook.id}+'.html'">
+                        <div class="layui-col-xs5 layui-col-sm3 layui-col-md3 layui-col-lg3">
+                            <img style=" width:100px; height:125px;"
+                                 th:src="${hotBook.picUrl}"/>
+                        </div>
+
+                        <div class="layui-col-xs7 layui-col-sm6 layui-col-md7 layui-col-lg7"
+                             style="float: left;padding-right: 10px">
+                            <div><b th:text="${hotBook.bookName}"></b></div>
+                            <div class="layui-col-xs8 layui-col-sm9 layui-col-md10 layui-col-lg10"
+                                 th:text="'作者：'+ ${hotBook.author}"></div>
+                            <div class="layui-col-xs3 layui-col-sm2 layui-col-md1 layui-col-lg1" style="text-align: right;">
+                            <b><i style="color: red"
+                                  th:text="${hotBook.score} + '分'"></i></b></div>
+                           </div>
+                            <div class="layui-elip layui-col-xs12 layui-col-sm9 layui-col-md10 layui-col-lg10 layui-col-sm9 layui-col-md10 layui-col-lg10"
+                                 th:text="${hotBook.bookDesc}">
+                            </div>
+
+
+
+                    </a>
+                </div>-->
+                </div>
+            </div>
+
+        </div>
+
+    </div>
+</div>
+</div>
+<div class="layui-colla-item"><a href="javascript:moreNewBooks()">
+    <h2 class="layui-colla-title">最近更新小说
+        <div style="float: right; margin-right: 20px"><i style="font-size: 14px;"
+                                                         class="layui-icon">更多&#xe65b;</i>
+        </div>
+    </h2>
+</a>
+    <div class="layui-colla-content layui-show">
+
+        <div class="layui-container" style="padding-left: 2px;padding-right: 5px">
+            <div class="layui-row">
+                <div th:each="newBook,bookStat : ${newBooks}" style="padding-bottom: 30px"
+                     class="layui-col-xs12 layui-col-sm6 layui-col-md6 layui-col-lg6">
+
+                    <a th:href="'/book/'+${newBook.id}+'.html'">
+                        <div class="line-limit-length layui-col-xs8 layui-col-sm6 layui-col-md6 layui-col-lg6"><b
+                                th:text="${bookStat.index}+1+'.'+${newBook.bookName}"></b> - <span class="layui-elip"
+                                                                                                   th:text="${newBook.author}"></span>
+                        </div>
+                        <div class="layui-col-sm3 layui-col-md3 layui-col-lg3"
+                             style="color: #FF5722;float: right;margin-right:5px"><i
+                                th:text="${#dates.format(newBook.updateTime, 'MM-dd HH:mm')}"></i></div>
+
+                        <div style="clear: both"></div>
+                        <div style="padding-left: 5px;padding-top: 5px"
+                             class="layui-elip layui-col-md11 layui-col-sm11 layui-col-lg11"
+                             th:text="'简介：'+ ${newBook.bookDesc}">
+
+                        </div>
+                    </a>
+                </div>
+
+
+            </div>
+        </div>
+
+    </div>
+</div>
+</div>
+<div th:replace="common/footer :: footer">
+</div>
+<div th:replace="common/js :: js">
+</div>
+
+<script src="/js/wap_collect.js"></script>
+<script>
+    (function () {
+        var bp = document.createElement('script');
+        var curProtocol = window.location.protocol.split(':')[0];
+        if (curProtocol === 'https') {
+            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
+        } else {
+            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
+        }
+        var s = document.getElementsByTagName("script")[0];
+        s.parentNode.insertBefore(bp, s);
+    })();
+</script>
+
+
+<script>
+
+    function moreNewBooks(event) {
+        window.location.href = "/book/search";
+    }
+
+    function searchBooks() {
+        var keywords = $("#title").val();
+        window.location.href = "/book/search?keyword=" + encodeURI(keywords);
+    }
+
+    function toMyCollect() {
+        var token = localStorage.getItem("token");
+        if (token) {
+            window.location.href = "/book/search?token=" + token;
+        } else {
+            window.location.href = "/user/login.html";
+        }
+    }
+
+</script>
+
+
+</body>
+</html>

script/crawlbook/application.yml

@@ -7,9 +7,9 @@ mybatis:
   type-aliases-package: xyz.zinglizingli.books.po
   configuration: {log-impl: org.apache.ibatis.logging.stdout.StdOutImpl}
 mysql: {charset: utf8mb4}
-books: {lowestScore: 9.0}
+books: {lowestScore: 6.0}
 crawl:
   website: {type: '4'}
-  soft-novel: '0'
-  manhua: '0'
+  soft-novel: 0
+  manhua: 0
 logging: {config: 'classpath:logback-boot.xml'}

sql/2020-04-21.sql

@@ -0,0 +1 @@
+alter table book_parse_log add column `priority` TINYINT(2) not null default 10 ;

sql/2020-04-22.sql

@@ -0,0 +1 @@
+alter table book_parse_log add column `update_count` TINYINT(2) not null default 0 ;

sql/books.sql

@@ -1,23 +1,4 @@
-/*
-Navicat MySQL Data Transfer
 
-Source Server         : aliyun_books
-Source Server Version : 80018
-Source Host           : 47.106.243.172:3306
-Source Database       : books
-
-Target Server Type    : MYSQL
-Target Server Version : 80018
-File Encoding         : 65001
-
-Date: 2019-11-15 06:10:36
-*/
-
-
-
--- ----------------------------
--- Table structure for `book`
--- ----------------------------
 DROP TABLE IF EXISTS `book`;
 CREATE TABLE `book` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
@@ -1042,3 +1023,8 @@ INSERT INTO `book_update_time_log` VALUES ('6', '6', NOW());
 INSERT INTO `book_update_time_log` VALUES ('7', '7', NOW());
 
 ALTER  TABLE  book_parse_log  drop  INDEX  uq_key_bookurl;
+
+alter table book_parse_log add column `priority` TINYINT(2) not null default 10 ;
+
+alter table book_parse_log add column `update_count` TINYINT(2) not null default 0 ;
+