更新版本号

README.md

@@ -1,7 +1,15 @@
-[![index](./assets/热门云产品1040.100.jpg)](https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console)
+[![index]( https://i.loli.net/2020/06/01/qF1uBMtswTIPJKx.jpg )]( https://www.eve.bz/cart.php?gid=11 )
 
 # 小说精品屋-plus
 
+#### 新项目：小说精品屋-微服务版
+
+基于小说精品屋-plus构建的Spring Cloud 微服务小说门户平台，可用于学习和商用。
+
+Gitee仓库地址： https://gitee.com/xiongxyang/novel-cloud
+
+GitHub仓库地址： https://github.com/201206030/novel-cloud
+
 #### 演示地址
 
 [点击前往](http://www.java2nb.com)（前台门户）
@@ -48,6 +56,7 @@ novel-plus -- 父工程
 | MyBatis Dynamic SQL  | Mybatis动态sql
 | PageHelper           | MyBatis分页插件
 | MyBatisGenerator     | 持久层代码生成插件
+| Sharding-Jdbc        | 代码层分库分表中间件
 | JJWT                 | JWT登录支持  
 | SpringSecurity       | 安全框架                           
 | Shiro                | 安全框架  

novel-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.1.2</version>
+        <version>2.6.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -57,6 +57,22 @@
             <version>${mybatis.version}</version>
         </dependency>
 
+        <!-- 分库分表-->
+        <!-- sharding jdbc依赖 -->
+
+        <dependency>
+            <groupId>io.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-spring-boot-starter</artifactId>
+            <version>${sharding.jdbc.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-spring-namespace</artifactId>
+            <version>${sharding.jdbc.version}</version>
+        </dependency>
+
+
         <!--分页助手启动器-->
         <dependency>
             <groupId>com.github.pagehelper</groupId>

novel-common/src/main/java/com/java2nb/novel/core/enums/ResponseStatus.java

@@ -55,7 +55,12 @@ public enum ResponseStatus {
      * */
     INVITE_CODE_INVALID(4001, "邀请码无效！"),
     AUTHOR_STATUS_FORBIDDEN(4002, "作者状态异常，暂不能管理小说！")
-    , BOOKNAME_EXISTS(4003,"已发布过同名小说！")
+    , BOOKNAME_EXISTS(4003,"已发布过同名小说！"),
+
+    /**
+     * 小说相关错误
+     */
+    BOOK_EXISTS(5001,"该小说已存在")
 
             ,
     /**
@@ -68,6 +73,7 @@ public enum ResponseStatus {
      * 其他通用错误
      * */
     PASSWORD_ERROR(88001,"密码错误！");
+
     private int code;
     private String msg;
 

novel-common/src/main/java/com/java2nb/novel/entity/CrawlSingleTask.java

@@ -10,9 +10,21 @@ public class CrawlSingleTask {
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     private Integer sourceId;
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    private String sourceName;
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     private String sourceBookId;
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    private Integer catId;
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    private String bookName;
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    private String authorName;
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     private Byte taskStatus;
 
@@ -42,6 +54,16 @@ public class CrawlSingleTask {
         this.sourceId = sourceId;
     }
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public String getSourceName() {
+        return sourceName;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public void setSourceName(String sourceName) {
+        this.sourceName = sourceName == null ? null : sourceName.trim();
+    }
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public String getSourceBookId() {
         return sourceBookId;
@@ -52,6 +74,36 @@ public class CrawlSingleTask {
         this.sourceBookId = sourceBookId == null ? null : sourceBookId.trim();
     }
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public Integer getCatId() {
+        return catId;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public void setCatId(Integer catId) {
+        this.catId = catId;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public String getBookName() {
+        return bookName;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public void setBookName(String bookName) {
+        this.bookName = bookName == null ? null : bookName.trim();
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public String getAuthorName() {
+        return authorName;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public void setAuthorName(String authorName) {
+        this.authorName = authorName == null ? null : authorName.trim();
+    }
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public Byte getTaskStatus() {
         return taskStatus;

novel-common/src/main/java/com/java2nb/novel/mapper/BookContentDynamicSqlSupport.java

@@ -24,7 +24,7 @@ public final class BookContentDynamicSqlSupport {
 
         public final SqlColumn<Long> indexId = column("index_id", JDBCType.BIGINT);
 
-        public final SqlColumn<String> content = column("content", JDBCType.LONGVARCHAR);
+        public final SqlColumn<String> content = column("content", JDBCType.VARCHAR);
 
         public BookContent() {
             super("book_content");

novel-common/src/main/java/com/java2nb/novel/mapper/BookContentMapper.java

@@ -63,7 +63,7 @@ public interface BookContentMapper {
     @Results(id="BookContentResult", value = {
         @Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
         @Result(column="index_id", property="indexId", jdbcType=JdbcType.BIGINT),
-        @Result(column="content", property="content", jdbcType=JdbcType.LONGVARCHAR)
+        @Result(column="content", property="content", jdbcType=JdbcType.VARCHAR)
     })
     List<BookContent> selectMany(SelectStatementProvider selectStatement);
 

novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskDynamicSqlSupport.java

@@ -16,9 +16,21 @@ public final class CrawlSingleTaskDynamicSqlSupport {
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public static final SqlColumn<Integer> sourceId = crawlSingleTask.sourceId;
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public static final SqlColumn<String> sourceName = crawlSingleTask.sourceName;
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public static final SqlColumn<String> sourceBookId = crawlSingleTask.sourceBookId;
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public static final SqlColumn<Integer> catId = crawlSingleTask.catId;
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public static final SqlColumn<String> bookName = crawlSingleTask.bookName;
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public static final SqlColumn<String> authorName = crawlSingleTask.authorName;
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public static final SqlColumn<Byte> taskStatus = crawlSingleTask.taskStatus;
 
@@ -34,8 +46,16 @@ public final class CrawlSingleTaskDynamicSqlSupport {
 
         public final SqlColumn<Integer> sourceId = column("source_id", JDBCType.INTEGER);
 
+        public final SqlColumn<String> sourceName = column("source_name", JDBCType.VARCHAR);
+
         public final SqlColumn<String> sourceBookId = column("source_book_id", JDBCType.VARCHAR);
 
+        public final SqlColumn<Integer> catId = column("cat_id", JDBCType.INTEGER);
+
+        public final SqlColumn<String> bookName = column("book_name", JDBCType.VARCHAR);
+
+        public final SqlColumn<String> authorName = column("author_name", JDBCType.VARCHAR);
+
         public final SqlColumn<Byte> taskStatus = column("task_status", JDBCType.TINYINT);
 
         public final SqlColumn<Byte> excCount = column("exc_count", JDBCType.TINYINT);

novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskMapper.java

@@ -35,7 +35,7 @@ import org.mybatis.dynamic.sql.util.mybatis3.MyBatis3Utils;
 @Mapper
 public interface CrawlSingleTaskMapper {
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
-    BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceBookId, taskStatus, excCount, createTime);
+    BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceName, sourceBookId, catId, bookName, authorName, taskStatus, excCount, createTime);
 
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     @SelectProvider(type=SqlProviderAdapter.class, method="select")
@@ -63,7 +63,11 @@ public interface CrawlSingleTaskMapper {
     @Results(id="CrawlSingleTaskResult", value = {
         @Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
         @Result(column="source_id", property="sourceId", jdbcType=JdbcType.INTEGER),
+        @Result(column="source_name", property="sourceName", jdbcType=JdbcType.VARCHAR),
         @Result(column="source_book_id", property="sourceBookId", jdbcType=JdbcType.VARCHAR),
+        @Result(column="cat_id", property="catId", jdbcType=JdbcType.INTEGER),
+        @Result(column="book_name", property="bookName", jdbcType=JdbcType.VARCHAR),
+        @Result(column="author_name", property="authorName", jdbcType=JdbcType.VARCHAR),
         @Result(column="task_status", property="taskStatus", jdbcType=JdbcType.TINYINT),
         @Result(column="exc_count", property="excCount", jdbcType=JdbcType.TINYINT),
         @Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP)
@@ -96,7 +100,11 @@ public interface CrawlSingleTaskMapper {
         return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c ->
             c.map(id).toProperty("id")
             .map(sourceId).toProperty("sourceId")
+            .map(sourceName).toProperty("sourceName")
             .map(sourceBookId).toProperty("sourceBookId")
+            .map(catId).toProperty("catId")
+            .map(bookName).toProperty("bookName")
+            .map(authorName).toProperty("authorName")
             .map(taskStatus).toProperty("taskStatus")
             .map(excCount).toProperty("excCount")
             .map(createTime).toProperty("createTime")
@@ -108,7 +116,11 @@ public interface CrawlSingleTaskMapper {
         return MyBatis3Utils.insertMultiple(this::insertMultiple, records, crawlSingleTask, c ->
             c.map(id).toProperty("id")
             .map(sourceId).toProperty("sourceId")
+            .map(sourceName).toProperty("sourceName")
             .map(sourceBookId).toProperty("sourceBookId")
+            .map(catId).toProperty("catId")
+            .map(bookName).toProperty("bookName")
+            .map(authorName).toProperty("authorName")
             .map(taskStatus).toProperty("taskStatus")
             .map(excCount).toProperty("excCount")
             .map(createTime).toProperty("createTime")
@@ -120,7 +132,11 @@ public interface CrawlSingleTaskMapper {
         return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c ->
             c.map(id).toPropertyWhenPresent("id", record::getId)
             .map(sourceId).toPropertyWhenPresent("sourceId", record::getSourceId)
+            .map(sourceName).toPropertyWhenPresent("sourceName", record::getSourceName)
             .map(sourceBookId).toPropertyWhenPresent("sourceBookId", record::getSourceBookId)
+            .map(catId).toPropertyWhenPresent("catId", record::getCatId)
+            .map(bookName).toPropertyWhenPresent("bookName", record::getBookName)
+            .map(authorName).toPropertyWhenPresent("authorName", record::getAuthorName)
             .map(taskStatus).toPropertyWhenPresent("taskStatus", record::getTaskStatus)
             .map(excCount).toPropertyWhenPresent("excCount", record::getExcCount)
             .map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime)
@@ -158,7 +174,11 @@ public interface CrawlSingleTaskMapper {
     static UpdateDSL<UpdateModel> updateAllColumns(CrawlSingleTask record, UpdateDSL<UpdateModel> dsl) {
         return dsl.set(id).equalTo(record::getId)
                 .set(sourceId).equalTo(record::getSourceId)
+                .set(sourceName).equalTo(record::getSourceName)
                 .set(sourceBookId).equalTo(record::getSourceBookId)
+                .set(catId).equalTo(record::getCatId)
+                .set(bookName).equalTo(record::getBookName)
+                .set(authorName).equalTo(record::getAuthorName)
                 .set(taskStatus).equalTo(record::getTaskStatus)
                 .set(excCount).equalTo(record::getExcCount)
                 .set(createTime).equalTo(record::getCreateTime);
@@ -168,7 +188,11 @@ public interface CrawlSingleTaskMapper {
     static UpdateDSL<UpdateModel> updateSelectiveColumns(CrawlSingleTask record, UpdateDSL<UpdateModel> dsl) {
         return dsl.set(id).equalToWhenPresent(record::getId)
                 .set(sourceId).equalToWhenPresent(record::getSourceId)
+                .set(sourceName).equalToWhenPresent(record::getSourceName)
                 .set(sourceBookId).equalToWhenPresent(record::getSourceBookId)
+                .set(catId).equalToWhenPresent(record::getCatId)
+                .set(bookName).equalToWhenPresent(record::getBookName)
+                .set(authorName).equalToWhenPresent(record::getAuthorName)
                 .set(taskStatus).equalToWhenPresent(record::getTaskStatus)
                 .set(excCount).equalToWhenPresent(record::getExcCount)
                 .set(createTime).equalToWhenPresent(record::getCreateTime);
@@ -178,7 +202,11 @@ public interface CrawlSingleTaskMapper {
     default int updateByPrimaryKey(CrawlSingleTask record) {
         return update(c ->
             c.set(sourceId).equalTo(record::getSourceId)
+            .set(sourceName).equalTo(record::getSourceName)
             .set(sourceBookId).equalTo(record::getSourceBookId)
+            .set(catId).equalTo(record::getCatId)
+            .set(bookName).equalTo(record::getBookName)
+            .set(authorName).equalTo(record::getAuthorName)
             .set(taskStatus).equalTo(record::getTaskStatus)
             .set(excCount).equalTo(record::getExcCount)
             .set(createTime).equalTo(record::getCreateTime)
@@ -190,7 +218,11 @@ public interface CrawlSingleTaskMapper {
     default int updateByPrimaryKeySelective(CrawlSingleTask record) {
         return update(c ->
             c.set(sourceId).equalToWhenPresent(record::getSourceId)
+            .set(sourceName).equalToWhenPresent(record::getSourceName)
             .set(sourceBookId).equalToWhenPresent(record::getSourceBookId)
+            .set(catId).equalToWhenPresent(record::getCatId)
+            .set(bookName).equalToWhenPresent(record::getBookName)
+            .set(authorName).equalToWhenPresent(record::getAuthorName)
             .set(taskStatus).equalToWhenPresent(record::getTaskStatus)
             .set(excCount).equalToWhenPresent(record::getExcCount)
             .set(createTime).equalToWhenPresent(record::getCreateTime)

novel-common/src/main/resources/application-common-dev.yml

@@ -1,11 +1,8 @@
 spring:
   profiles:
     include: [common]
-  datasource:
+  main:
-    url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
+    allow-bean-definition-overriding: true
-    username: root
-    password: test123456
-    driver-class-name: com.mysql.cj.jdbc.Driver
   #Redis服务器IP
   redis:
     host: 127.0.0.1
@@ -25,6 +22,51 @@ spring:
         min-idle: 0
     #连接超时时间（毫秒）
     timeout: 30000
+  datasource:
+    url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
+    username: root
+    password: test123456
+    driver-class-name: com.mysql.cj.jdbc.Driver
+
+####使用shardingJdbc时，
+####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误
+##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR
+
+sharding:
+  jdbc:
+    datasource:
+      names: ds0 #,ds1
+      ds0:
+        type: com.zaxxer.hikari.HikariDataSource
+        driver-class-name: com.mysql.cj.jdbc.Driver
+        jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
+        username: root
+        password: test123456
+    #      ds1:
+    #        type: com.alibaba.druid.pool.DruidDataSource
+    #        driver-class-name: com.mysql.jdbc.Driver
+    #        url: jdbc:mysql://localhost:3306/novel_plus2
+    #        username: root
+    #        password: test123456
+    config:
+      sharding:
+        props:
+          sql.show: true
+        tables:
+          book_content:  #book_content表
+            key-generator-column-name: id  #主键
+            actual-data-nodes: ds${0}.book_content${0..9}  #数据节点
+            #            database-strategy:  #分库策略
+            #              inline:
+            #                sharding-column: book_id
+            #                algorithm-expression: ds${book_id % 10}
+            table-strategy:  #分表策略
+              inline:
+                shardingColumn: index_id
+                algorithm-expression: book_content${index_id % 10}
+
+
+
 
 
 
@@ -33,7 +75,3 @@ pic:
     type: 2 #图片保存方式, 1不保存，使用爬取的网络图片 ，2保存在自己的存储介质
     storage: local #存储介质，local：本地，OSS：阿里云对象存储，fastDfs：分布式文件系统
     path: /var/pic  #图片保存路径
-
-
-
-

novel-common/src/main/resources/mybatis/generatorConfig.xml

@@ -9,7 +9,7 @@
         </commentGenerator>
         <jdbcConnection
                 connectionURL="jdbc:mysql://127.0.0.1:3306/novel_plus?tinyInt1isBit=false&amp;useUnicode=true&amp;characterEncoding=utf-8&amp;serverTimezone=Asia/Shanghai&amp;nullCatalogMeansCurrent=true"
-                driverClass="com.mysql.jdbc.Driver" password=""
+                driverClass="com.mysql.jdbc.Driver" password="test123456"
                 userId="root" />
 
         <!-- 默认false，把JDBC DECIMAL 和 NUMERIC 类型解析为 Integer， 为 true时把JDBC DECIMAL
@@ -44,7 +44,7 @@
         </javaClientGenerator>
 
         <!--生成全部表tableName设为%-->
-        <table tableName="user_buy_record" domainObjectName="UserBuyRecord"/>
+        <table tableName="crawl_single_task"/>
 
         <!-- 指定数据库表 -->
         <!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->

novel-crawl/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.1.2</version>
+        <version>2.6.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -25,6 +25,8 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-security</artifactId>
         </dependency>
+
+
     </dependencies>
 
     <build>

novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java

@@ -3,11 +3,12 @@ package com.java2nb.novel.controller;
 import com.github.pagehelper.PageInfo;
 import com.java2nb.novel.core.bean.ResultBean;
 import com.java2nb.novel.core.utils.BeanUtil;
+import com.java2nb.novel.entity.CrawlSingleTask;
 import com.java2nb.novel.entity.CrawlSource;
 import com.java2nb.novel.service.CrawlService;
+import com.java2nb.novel.vo.CrawlSingleTaskVO;
 import com.java2nb.novel.vo.CrawlSourceVO;
 import lombok.RequiredArgsConstructor;
-import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -56,6 +57,38 @@ public class CrawlController {
         return ResultBean.ok();
     }
 
+    /**
+     * 新增单本采集任务
+     * */
+    @PostMapping("addCrawlSingleTask")
+    public ResultBean addCrawlSingleTask(CrawlSingleTask singleTask){
+        crawlService.addCrawlSingleTask(singleTask);
+
+        return ResultBean.ok();
+
+    }
+
+    /**
+     * 单本采集任务分页列表查询
+     * */
+    @PostMapping("listCrawlSingleTaskByPage")
+    public ResultBean listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
+
+        return ResultBean.ok(new PageInfo<>(BeanUtil.copyList(crawlService.listCrawlSingleTaskByPage(page,pageSize), CrawlSingleTaskVO.class)
+        ));
+    }
+
+    /**
+     * 删除采集任务
+     * */
+    @PostMapping("delCrawlSingleTask")
+    public ResultBean delCrawlSingleTask(Long id){
+
+        crawlService.delCrawlSingleTask(id);
+
+        return ResultBean.ok();
+    }
+
 
 
 

novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java

@@ -96,8 +96,12 @@ public class CrawlParser {
 
                     String desc = bookDetailHtml.substring(bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length());
                     desc = desc.substring(0, desc.indexOf(ruleBean.getDescEnd()));
-                    //过滤掉简介中的a标签
+                    //过滤掉简介中的特殊标签
-                    desc = desc.replaceAll("<a[^<]+</a>","");
+                    desc = desc.replaceAll("<a[^<]+</a>","")
+                            .replaceAll("<font[^<]+</font>","")
+                            .replaceAll("<p>\\s*</p>","")
+                            .replaceAll("<p>","")
+                            .replaceAll("</p>","<br/>");
                     //设置书籍简介
                     book.setBookDesc(desc);
                     if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {
@@ -237,8 +241,6 @@ public class CrawlParser {
                         }
 
 
-                    } else {
-                        break;
                     }
 
 

novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java

@@ -3,10 +3,7 @@ package com.java2nb.novel.core.listener;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.java2nb.novel.core.crawl.CrawlParser;
 import com.java2nb.novel.core.crawl.RuleBean;
-import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.entity.*;
-import com.java2nb.novel.entity.BookContent;
-import com.java2nb.novel.entity.BookIndex;
-import com.java2nb.novel.entity.CrawlSource;
 import com.java2nb.novel.service.BookService;
 import com.java2nb.novel.service.CrawlService;
 import com.java2nb.novel.utils.Constants;
@@ -40,9 +37,9 @@ public class StarterListener implements ServletContextListener {
 
     @Override
     public void contextInitialized(ServletContextEvent sce) {
-        log.info("程序启动,开始执行自动更新线程。。。");
         for (int i = 0; i < updateThreadCount; i++) {
             new Thread(() -> {
+                log.info("程序启动,开始执行自动更新线程。。。");
                 while (true) {
                     try {
                         //1.查询最新目录更新时间在一个月之内的前100条需要更新的数据
@@ -83,6 +80,42 @@ public class StarterListener implements ServletContextListener {
 
                 }
             }).start();
-        }
+
+
+        }
+
+
+        new Thread(() -> {
+            log.info("程序启动,开始执行单本采集任务线程。。。");
+            while (true) {
+                CrawlSingleTask task = null;
+                byte crawlStatus = 0;
+                try {
+                    //获取采集任务
+                    task = crawlService.getCrawlSingleTask();
+
+                    if (task != null) {
+                        //查询爬虫规则
+                        CrawlSource source = crawlService.queryCrawlSource(task.getSourceId());
+                        RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
+
+                        if (crawlService.parseBookAndSave(task.getCatId(), ruleBean, task.getSourceId(), task.getSourceBookId())) {
+                            //采集成功
+                            crawlStatus = 1;
+                        }
+
+                    }
+
+                    Thread.sleep(1000 * 60);
+
+                } catch (Exception e) {
+                    log.error(e.getMessage(), e);
+                }
+                if (task != null) {
+                    crawlService.updateCrawlSingleTask(task, crawlStatus);
+                }
+
+            }
+        }).start();
     }
 }

novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java

@@ -1,6 +1,7 @@
 package com.java2nb.novel.service;
 
 import com.java2nb.novel.core.crawl.RuleBean;
+import com.java2nb.novel.entity.CrawlSingleTask;
 import com.java2nb.novel.entity.CrawlSource;
 
 import java.util.List;
@@ -39,6 +40,16 @@ public interface CrawlService {
      * */
     void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus);
 
+    /**
+     * 采集并保存小说
+     * @param catId 分类ID
+     * @param bookId 小说ID
+     * @param sourceId 源ID
+     * @param ruleBean 采集规则\
+     * @return true:成功，false:失败
+     * */
+    boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId);
+
     /**
      * 根据爬虫状态查询爬虫源集合
      * @param sourceStatus 状态，0关闭，1开启
@@ -61,4 +72,37 @@ public interface CrawlService {
      * @return 源信息
      * */
     CrawlSource queryCrawlSource(Integer sourceId);
+
+    /**
+     * 新增单本采集任务
+     * @param singleTask 任务信息对象
+     * */
+    void addCrawlSingleTask(CrawlSingleTask singleTask);
+
+    /**
+     * 单本采集任务分页列表查询
+     * @param page 当前页码
+     * @param pageSize 分页大小
+     * @return 单本采集任务集合
+     * */
+    List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize);
+
+    /**
+     * 删除采集任务
+     * @param id 任务ID
+     * */
+    void delCrawlSingleTask(Long id);
+
+    /**
+     * 获取采集任务
+     * @return 采集任务
+     * */
+    CrawlSingleTask getCrawlSingleTask();
+
+    /**
+     * 更新单本采集任务
+     * @param task 采集任务
+     * @param status 采集状态
+     * */
+    void updateCrawlSingleTask(CrawlSingleTask task, Byte status);
 }

novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java

@@ -6,12 +6,12 @@ import com.java2nb.novel.core.cache.CacheKey;
 import com.java2nb.novel.core.cache.CacheService;
 import com.java2nb.novel.core.crawl.CrawlParser;
 import com.java2nb.novel.core.crawl.RuleBean;
+import com.java2nb.novel.core.enums.ResponseStatus;
+import com.java2nb.novel.core.exception.BusinessException;
 import com.java2nb.novel.core.utils.IdWorker;
 import com.java2nb.novel.core.utils.SpringUtil;
 import com.java2nb.novel.core.utils.ThreadUtil;
-import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.entity.*;
-import com.java2nb.novel.entity.BookContent;
-import com.java2nb.novel.entity.BookIndex;
 import com.java2nb.novel.entity.CrawlSource;
 import com.java2nb.novel.mapper.*;
 import com.java2nb.novel.service.BookService;
@@ -33,8 +33,7 @@ import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClient;
 import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlBookId;
 import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlSourceId;
 import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*;
-import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
+import static org.mybatis.dynamic.sql.SqlBuilder.*;
-import static org.mybatis.dynamic.sql.SqlBuilder.update;
 import static org.mybatis.dynamic.sql.select.SelectDSL.select;
 
 /**
@@ -48,6 +47,8 @@ public class CrawlServiceImpl implements CrawlService {
 
     private final CrawlSourceMapper crawlSourceMapper;
 
+    private final CrawlSingleTaskMapper crawlSingleTaskMapper;
+
     private final BookService bookService;
 
 
@@ -140,6 +141,62 @@ public class CrawlServiceImpl implements CrawlService {
         return crawlSourceMapper.selectMany(render).get(0);
     }
 
+    @Override
+    public void addCrawlSingleTask(CrawlSingleTask singleTask) {
+
+        if(bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(),singleTask.getAuthorName())){
+            throw new BusinessException(ResponseStatus.BOOK_EXISTS);
+
+        }
+        singleTask.setCreateTime(new Date());
+        crawlSingleTaskMapper.insertSelective(singleTask);
+
+
+    }
+
+    @Override
+    public List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize) {
+        PageHelper.startPage(page, pageSize);
+        SelectStatementProvider render = select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
+                .from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
+                .orderBy(CrawlSingleTaskDynamicSqlSupport.createTime.descending())
+                .build()
+                .render(RenderingStrategies.MYBATIS3);
+        return crawlSingleTaskMapper.selectMany(render);
+    }
+
+    @Override
+    public void delCrawlSingleTask(Long id) {
+        crawlSingleTaskMapper.deleteByPrimaryKey(id);
+    }
+
+    @Override
+    public CrawlSingleTask getCrawlSingleTask() {
+
+         List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
+                .from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
+                .where(CrawlSingleTaskDynamicSqlSupport.taskStatus,isEqualTo((byte)2))
+                 .orderBy(CrawlSingleTaskDynamicSqlSupport.createTime)
+                 .limit(1)
+                .build()
+                .render(RenderingStrategies.MYBATIS3));
+
+         return list.size() > 0 ? list.get(0) : null;
+    }
+
+    @Override
+    public void updateCrawlSingleTask(CrawlSingleTask task, Byte status) {
+        byte excCount = task.getExcCount();
+        excCount+=1;
+        task.setExcCount(excCount);
+        if(status == 1 || excCount == 5){
+            //当采集成功或者采集次数等于5，则更新采集最终状态，并停止采集
+            task.setTaskStatus(status);
+        }
+        crawlSingleTaskMapper.updateByPrimaryKeySelective(task);
+
+    }
+
     /**
      * 解析分类列表
      */
@@ -173,35 +230,7 @@ public class CrawlServiceImpl implements CrawlService {
 
 
                                 String bookId = bookIdMatcher.group(1);
-                                Book book = CrawlParser.parseBook(ruleBean, bookId);
+                                parseBookAndSave(catId, ruleBean, sourceId, bookId);
-                                //这里只做新书入库，查询是否存在这本书
-                                Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
-                                //如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库
-                                if (existBook == null) {
-                                    //没有该书，可以入库
-                                    book.setCatId(catId);
-                                    //根据分类ID查询分类
-                                    book.setCatName(bookService.queryCatNameByCatId(catId));
-                                    if (catId == 7) {
-                                        //女频
-                                        book.setWorkDirection((byte) 1);
-                                    } else {
-                                        //男频
-                                        book.setWorkDirection((byte) 0);
-                                    }
-                                    book.setCrawlBookId(bookId);
-                                    book.setCrawlSourceId(sourceId);
-                                    book.setCrawlLastTime(new Date());
-                                    book.setId(new IdWorker().nextId());
-                                    //解析章节目录
-                                    Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
-
-                                    bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
-
-                                } else {
-                                    //只更新书籍的爬虫相关字段
-                                    bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
-                                }
                             } catch (Exception e) {
                                 log.error(e.getMessage(), e);
                             }
@@ -232,6 +261,43 @@ public class CrawlServiceImpl implements CrawlService {
 
     }
 
+    @Override
+    public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) {
+        Book book = CrawlParser.parseBook(ruleBean, bookId);
+        if(book.getBookName() == null || book.getAuthorName() == null){
+            return false;
+        }
+        //这里只做新书入库，查询是否存在这本书
+        Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
+        //如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库
+        if (existBook == null) {
+            //没有该书，可以入库
+            book.setCatId(catId);
+            //根据分类ID查询分类
+            book.setCatName(bookService.queryCatNameByCatId(catId));
+            if (catId == 7) {
+                //女频
+                book.setWorkDirection((byte) 1);
+            } else {
+                //男频
+                book.setWorkDirection((byte) 0);
+            }
+            book.setCrawlBookId(bookId);
+            book.setCrawlSourceId(sourceId);
+            book.setCrawlLastTime(new Date());
+            book.setId(new IdWorker().nextId());
+            //解析章节目录
+            Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
+
+            bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
+
+        } else {
+            //只更新书籍的爬虫相关字段
+            bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
+        }
+        return true;
+    }
+
     @Override
     public void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus) {
         CrawlSource source = new CrawlSource();

novel-crawl/src/main/java/com/java2nb/novel/vo/CrawlSingleTaskVO.java

@@ -0,0 +1,26 @@
+package com.java2nb.novel.vo;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.java2nb.novel.entity.CrawlSingleTask;
+import com.java2nb.novel.entity.CrawlSource;
+import lombok.Data;
+
+import java.util.Date;
+
+/**
+ * @author Administrator
+ */
+@Data
+public class CrawlSingleTaskVO extends CrawlSingleTask {
+
+    @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm")
+    private Date createTime;
+
+
+
+
+    @Override
+    public String toString() {
+        return super.toString();
+    }
+}

novel-crawl/src/main/resources/static/css/user.css

@@ -10,7 +10,7 @@
 .user_l .log_list { width:350px }
 .user_l .s_input { margin-bottom:25px; font-size:14px }
 .s_input { width:348px; height:30px; line-height:38px\9; vertical-align:middle; border:1px solid #ddd; border-radius:2px }
-.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px; background:url(../images/icon_user.png) no-repeat 13px 13px }
+.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px}
 .icon_key { background-position: 13px -51px }
 .icon_code { background-position: 13px -117px; width:200px; float:left }
 .code_pic { height:38px; float:right }
@@ -37,7 +37,7 @@
 .fast_tit .title { background:#fff; font-size:16px; padding:3px 14px; position:relative; display:inline-block; z-index:999 }
 /*userinfo*/
 .my_l { width:198px; float:left; font-size: 13px; padding-top: 20px; }
-.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff; background:url(../images/icon_user.png) no-repeat; margin-bottom:5px; color: #666 }
+.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff;  margin-bottom:5px; color: #666 }
 .my_l li .on { background-color:#fafafa; border-left:2px solid #f80; color:#000; border-radius: 0 2px 2px 0 }
 .my_l .link_1 { background-position:32px -188px }
 .my_l .link_2 { background-position:32px -230px }

novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_add.html

@@ -0,0 +1,188 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
+        <title>爬虫管理系统-小说精品屋</title>
+        <link rel="stylesheet" href="/css/base.css?v=1"/>
+        <link rel="stylesheet" href="/css/user.css"/>
+    </head>
+</head>
+<body class="">
+
+<div class="header">
+    <div class="mainNav" id="mainNav">
+        <div class="box_center cf"
+             style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
+
+            小说精品屋爬虫管理
+
+        </div>
+    </div>
+</div>
+
+<div class="main box_center cf">
+    <div class="userBox cf">
+        <div class="my_l">
+
+            <ul class="log_list">
+                <li><a class="link_1" href="/">爬虫源管理</a></li>
+                <li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
+                <!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
+                <li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
+            </ul>
+
+        </div>
+        <div class="my_r">
+            <div class="my_bookshelf">
+
+                <div class="userBox cf">
+                    <form method="post" action="./register.html" id="form2">
+
+                        <div class="user_l">
+                            <div></div>
+                            <h3>采集信息填写（示例均为笔趣阁：http://www.mcmssc.com）</h3>
+                            <ul class="log_list">
+                                <li><span id="LabErr"></span></li>
+                                <input type="hidden" id="sourceName" class="s_input icon_name" >
+                                <b>采集源:</b>
+                                <li><select id="sourceId" class="s_input icon_key">
+                                </select></li>
+                                <b>采集分类:</b>
+                                <li><select id="catId" class="s_input icon_key">
+                                    <option value="1">玄幻奇幻</option>
+                                    <option value="2">武侠仙侠</option>
+                                    <option value="3">都市言情</option>
+                                    <option value="4">历史军事</option>
+                                    <option value="5">科幻灵异</option>
+                                    <option value="6">网游竞技</option>
+                                    <option value="7">女生频道</option>
+                                </select>
+                                </li>
+                                示例：<b>73_73911</b>
+                                <li><input type="text" id="sourceBookId" class="s_input icon_key"
+                                           placeholder="采集的源站小说ID"/></li>
+                                示例：<b>苏厨</b>
+                                <li><input type="text" id="bookName" class="s_input icon_key"
+                                           placeholder="采集的小说名"/></li>
+                                示例：<b>二子从周</b>
+                                <li><input type="text" id="authorName" class="s_input icon_key"
+                                           placeholder="采集的小说作者名"/></li>
+
+                                <!--示例：<b>https://m.xdingdiann.com/sort/0/1.html</b>
+                                <li><input type="text" id="updateBookListUrl" class="s_input icon_key"
+                                           placeholder="小说更新列表url"></li>-->
+
+
+                                <li><input type="button" onclick="addCrawlSingleTask()" name="btnRegister" value="提交"
+                                           id="btnRegister" class="btn_red"></li>
+                            </ul>
+                        </div>
+                    </form>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+
+</body>
+<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
+<script src="/layui/layui.all.js" type="text/javascript"></script>
+<script src="/javascript/header.js" type="text/javascript"></script>
+<script src="/javascript/user.js" type="text/javascript"></script>
+<script language="javascript" type="text/javascript">
+    $(function () {
+        $.ajax({
+            type: "POST",
+            url: "/crawl/listCrawlByPage",
+            data: {'curr':1,'limit':100},
+            dataType: "json",
+            success: function (data) {
+                if (data.code == 200) {
+                    var crawlSourceList = data.data.list;
+                    if (crawlSourceList.length > 0) {
+                        var crawlSourceListHtml = "";
+                        for(var i=0;i<crawlSourceList.length;i++){
+                            var crawlSource = crawlSourceList[i];
+                            crawlSourceListHtml+=("<option value=\""+crawlSource.id+"\">"+crawlSource.sourceName+"</option>");
+                        }
+                        $("#sourceId").html(crawlSourceListHtml);
+
+
+                    }
+
+
+                } else if (data.code == 1001) {
+                    //未登录
+                    location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href);
+
+                }else {
+                    layer.alert(data.msg);
+                }
+
+            },
+            error: function () {
+                layer.alert('网络异常');
+            }
+        })
+    });
+
+
+    function addCrawlSingleTask() {
+
+        var sourceId = $("#sourceId").find("option:selected").val();
+
+        var catId = $("#catId").find("option:selected").val();
+
+        var sourceName = $("#sourceId").find("option:selected").html();
+
+
+        var sourceBookId = $("#sourceBookId").val();
+
+        if (!sourceBookId){
+            layer.alert("采集的源站小说ID不能为空");
+            return;
+        }
+
+        var bookName = $("#bookName").val();
+
+        if (!bookName){
+            layer.alert("采集的小说名不能为空");
+            return;
+        }
+
+        var authorName = $("#authorName").val();
+
+        if (!authorName){
+            layer.alert("采集的小说作者名不能为空");
+            return;
+        }
+
+        $.ajax({
+            type: "POST",
+            url: "/crawl/addCrawlSingleTask",
+            data: {'sourceId': sourceId, 'sourceName': sourceName,'catId':catId,'sourceBookId' : sourceBookId,'bookName':bookName,'authorName':authorName},
+            dataType: "json",
+            success: function (data) {
+                if (data.code == 200) {
+
+                    window.location.href = '/crawl/crawlSingleTask_list.html';
+
+
+                } else {
+                    layer.alert(data.msg);
+                }
+
+            },
+            error: function () {
+                layer.alert('网络异常');
+            }
+        })
+
+    }
+
+</script>
+</html>

novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html

@@ -0,0 +1,230 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
+        <title>爬虫管理系统-小说精品屋</title>
+        <link rel="stylesheet" href="/css/base.css?v=1"/>
+        <link rel="stylesheet" href="/css/user.css"/>
+    </head>
+</head>
+<body class="">
+
+<div class="header">
+    <div class="mainNav" id="mainNav">
+        <div class="box_center cf"
+             style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
+
+            小说精品屋爬虫管理
+
+        </div>
+    </div>
+</div>
+
+<div class="main box_center cf">
+    <div class="userBox cf">
+        <div class="my_l">
+
+            <ul class="log_list">
+                <li><a class="link_1" href="/">爬虫源管理</a></li>
+                <li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
+                <!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
+                 <li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
+            </ul>
+
+        </div>
+        <div class="my_r">
+            <div class="my_bookshelf">
+                <div class="title cf">
+                    <h2 class="fl">单本采集任务列表</h2>
+                    <div class="fr"><a href="/crawl/crawlSingleTask_add.html" class="btn_red">增加单本采集任务</a></div>
+                </div>
+
+                <div id="divData" class="updateTable">
+                    <table cellpadding="0" cellspacing="0">
+                        <thead>
+                        <tr>
+                            <th class="style">
+                                序号
+                            </th>
+                            <th class="name">
+                                采集小说名
+                            </th>
+                            <th class="name">
+                                采集小说作者名
+                            </th>
+                            <th class="goread">
+                                采集次数
+                            </th>
+                            <th class="goread">
+                                状态
+                            </th>
+                            <th class="name">
+                                创建时间
+                            </th>
+                            <th class="goread">
+                                操作
+                            </th>
+                        </tr>
+                        </thead>
+                        <tbody id="crawlSourceList">
+
+
+                        </tbody>
+                    </table>
+                    <div class="pageBox cf" id="shellPage">
+                    </div>
+                </div>
+                <!--<div id="divData" class="updateTable">
+                    <table cellpadding="0" cellspacing="0">
+                        <thead>
+                        <tr>
+
+                            <th class="name">
+                                爬虫源（已开启的爬虫源）
+                            </th>
+                            <th class="chapter">
+                                成功爬取数量（websocket实现）
+                            </th>
+                            <th class="time">
+                            目标爬取数量
+                            </th>
+                            <th class="goread">
+                                状态（正在运行，已停止）（一次只能运行一个爬虫源）
+                            </th>
+                            <th class="goread">
+                                操作（启动，停止）
+                            </th>
+                        </tr>
+                        </thead>
+                        <tbody id="bookShelfList">
+
+
+
+                        </tbody>
+                    </table>
+                    <div class="pageBox cf" id="shellPage">
+                    </div>
+                </div>-->
+            </div>
+        </div>
+    </div>
+</div>
+
+
+</body>
+<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
+<script src="/layui/layui.all.js" type="text/javascript"></script>
+<script src="/javascript/header.js" type="text/javascript"></script>
+<script src="/javascript/user.js" type="text/javascript"></script>
+<script language="javascript" type="text/javascript">
+    search(1, 10);
+
+    function search(curr, limit) {
+
+        $.ajax({
+            type: "POST",
+            url: "/crawl/listCrawlSingleTaskByPage",
+            data: {'curr': curr, 'limit': limit},
+            dataType: "json",
+            success: function (data) {
+                if (data.code == 200) {
+                    var crawlSourceList = data.data.list;
+                    if (crawlSourceList.length > 0) {
+                        var crawlSourceListHtml = "";
+                        for (var i = 0; i < crawlSourceList.length; i++) {
+                            var crawlSource = crawlSourceList[i];
+                            crawlSourceListHtml += (" <tr class=\"book_list\" vals=\"291\">\n" +
+                                "                            <td class=\"style bookclass\">\n" +
+                                "                                [" + (i + 1) + "]\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"name\">\n" +
+                                "                                " + crawlSource.bookName + "\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"name\">\n" +
+                                "                                " + crawlSource.authorName + "\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"goread\">\n" +
+                                "                                " + crawlSource.excCount + "\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"goread\">\n" +
+                                "                                " + (crawlSource.taskStatus == 0 ? '采集失败' : (crawlSource.taskStatus == 1 ? '采集成功' : (crawlSource.excCount>0?'采集中':'排队中'))) + "\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"name\" valsc=\"291|2037554|1\">"
+                                + crawlSource.createTime + "</td>\n" +
+
+                                " <td class=\"goread\" id='opt" + crawlSource.id + "'><a href='javascript:del(" + crawlSource.id + ")'>删除  </a></td>                     </tr>");
+                        }
+                        $("#crawlSourceList").html(crawlSourceListHtml);
+
+                        layui.use('laypage', function () {
+                            var laypage = layui.laypage;
+
+                            //执行一个laypage实例
+                            laypage.render({
+                                elem: 'shellPage' //注意，这里的 test1 是 ID，不用加 # 号
+                                , count: data.data.total //数据总数，从服务端得到,
+                                , curr: data.data.pageNum
+                                , limit: data.data.pageSize
+                                , jump: function (obj, first) {
+
+
+                                    //obj包含了当前分页的所有参数，比如：
+                                    console.log(obj.curr); //得到当前页，以便向服务端请求对应页的数据。
+                                    console.log(obj.limit); //得到每页显示的条数
+
+
+                                    //首次不执行
+                                    if (!first) {
+                                        search(obj.curr, obj.limit);
+                                    } else {
+
+                                    }
+                                }
+                            });
+                        });
+
+                    }
+
+
+                }  else {
+                    layer.alert(data.msg);
+                }
+
+            },
+            error: function () {
+                layer.alert('网络异常');
+            }
+        })
+
+    }
+
+
+    function del(id) {
+
+        $.ajax({
+            type: "POST",
+            url: "/crawl/delCrawlSingleTask",
+            data: {'id': id},
+            dataType: "json",
+            success: function (data) {
+                if (data.code == 200) {
+                    location.reload();
+
+
+                } else {
+                    layer.alert(data.msg);
+                }
+
+            },
+            error: function () {
+                layer.alert('网络异常');
+            }
+        })
+
+
+    }
+</script>
+</html>

novel-crawl/src/main/resources/templates/crawl/crawlSource_add.html

@@ -29,6 +29,7 @@
 
             <ul class="log_list">
                 <li><a class="link_1 on" href="/">爬虫源管理</a></li>
+                <li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
                 <!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
                 <li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
             </ul>

novel-crawl/src/main/resources/templates/crawl/crawlSource_list.html

@@ -28,6 +28,7 @@
 
             <ul class="log_list">
                 <li><a class="link_1 on" href="/">爬虫源管理</a></li>
+                <li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
                <!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
                 <li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
             </ul>

novel-front/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.1.2</version>
+        <version>2.6.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -43,24 +43,19 @@
         <dependency>
             <groupId>com.aliyun.oss</groupId>
             <artifactId>aliyun-sdk-oss</artifactId>
-            <version>2.4.0</version>
+            <version>${aliyun-sdk-oss.version}</version>
         </dependency>
         <dependency>
             <groupId>commons-fileupload</groupId>
             <artifactId>commons-fileupload</artifactId>
-            <version>1.3.1</version>
+            <version>${commons-fileupload.version}</version>
         </dependency>
 
         <!--FastDfs-->
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-mock</artifactId>
-            <version>2.0.8</version>
-        </dependency>
         <dependency>
             <groupId>com.github.tobato</groupId>
             <artifactId>fastdfs-client</artifactId>
-            <version>1.26.1-RELEASE</version>
+            <version>${fastdfs-client.version}</version>
         </dependency>
 
         <dependency>
@@ -69,6 +64,8 @@
         </dependency>
 
 
+
+
         <!--引入redisson分布式锁-->
        <!-- <dependency>
             <groupId>org.redisson</groupId>
@@ -80,7 +77,7 @@
         <dependency>
             <groupId>com.alipay.sdk</groupId>
             <artifactId>alipay-sdk-java</artifactId>
-            <version>4.9.153.ALL</version>
+            <version>${alipay-sdk-java.version}</version>
         </dependency>
     </dependencies>
 

novel-front/src/main/java/com/java2nb/novel/FrontNovelApplication.java

@@ -1,6 +1,7 @@
 package com.java2nb.novel;
 
 import com.github.tobato.fastdfs.FdfsClientConfig;
+import io.shardingsphere.shardingjdbc.spring.boot.SpringBootConfiguration;
 import org.mybatis.spring.annotation.MapperScan;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.java2nb</groupId>
     <artifactId>novel</artifactId>
-    <version>2.1.2</version>
+    <version>2.6.0</version>
     <modules>
         <module>novel-common</module>
         <module>novel-front</module>
@@ -40,6 +40,11 @@
         <jest.version>6.3.1</jest.version>
         <redis.version>1.4.1.RELEASE</redis.version>
         <redisson.version>3.12.5</redisson.version>
+        <sharding.jdbc.version>3.0.0</sharding.jdbc.version>
+        <aliyun-sdk-oss.version>2.4.0</aliyun-sdk-oss.version>
+        <commons-fileupload.version>1.3.1</commons-fileupload.version>
+        <fastdfs-client.version>1.26.1-RELEASE</fastdfs-client.version>
+        <alipay-sdk-java.version>4.9.153.ALL</alipay-sdk-java.version>
     </properties>
 
     <dependencyManagement>

sql/20200608.sql

@@ -0,0 +1,2 @@
+INSERT INTO `crawl_source` (`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES
+(6, '新笔趣阁', '{\n	\"bookListUrl\": \"http://www.xbiquge.la/fenlei/{catId}_{page}.html\",\n	\"catIdRule\": {\n		\"catId1\": \"1\",\n		\"catId2\": \"2\",\n		\"catId3\": \"3\",\n		\"catId4\": \"4\",\n		\"catId5\": \"6\",\n		\"catId6\": \"5\"\n	},\n	\"bookIdPatten\": \"<a\\\\s+href=\\\"http://www.xbiquge.la/(\\\\d+/\\\\d+)/\\\"\\\\s+target=\\\"_blank\\\">\",\n	\"pagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">(\\\\d+)/\\\\d+</em>\",\n	\"totalPagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">\\\\d+/(\\\\d+)</em>\",\n	\"bookDetailUrl\": \"http://www.xbiquge.la/{bookId}/\",\n	\"bookNamePatten\": \"<h1>([^/]+)</h1>\",\n	\"authorNamePatten\": \"者：([^/]+)</p>\",\n	\"picUrlPatten\": \"src=\\\"(http://www.xbiquge.la/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n	\"bookStatusRule\": {},\n	\"descStart\": \"<div id=\\\"intro\\\">\",\n	\"descEnd\": \"</div>\",\n	\"upadateTimePatten\": \"<p>最后更新：(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)</p>\",\n	\"upadateTimeFormatPatten\": \"yyyy-MM-dd HH:mm:ss\",\n	\"bookIndexUrl\": \"http://www.xbiquge.la/{bookId}/\",\n	\"indexIdPatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/(\\\\d+)\\\\.html\'\\\\s+>[^/]+</a>\",\n	\"indexNamePatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/\\\\d+\\\\.html\'\\\\s+>([^/]+)</a>\",\n	\"bookContentUrl\": \"http://www.xbiquge.la/{bookId}/{indexId}.html\",\n	\"contentStart\": \"<div id=\\\"content\\\">\",\n	\"contentEnd\": \"<p>\"\n}', 0, '2020-05-23 22:46:58', '2020-05-23 22:46:58');

sql/20200615.sql

@@ -0,0 +1,40 @@
+/*
+Navicat MySQL Data Transfer
+
+Source Server         : localhost
+Source Server Version : 50725
+Source Host           : localhost:3306
+Source Database       : novel_plus
+
+Target Server Type    : MYSQL
+Target Server Version : 50725
+File Encoding         : 65001
+
+Date: 2020-06-15 15:06:55
+*/
+
+SET FOREIGN_KEY_CHECKS=0;
+
+-- ----------------------------
+-- Table structure for crawl_single_task
+-- ----------------------------
+DROP TABLE IF EXISTS `crawl_single_task`;
+CREATE TABLE `crawl_single_task` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `source_id` int(11) DEFAULT NULL COMMENT '爬虫源ID',
+  `source_name` varchar(50) DEFAULT NULL COMMENT '爬虫源名',
+  `source_book_id` varchar(255) DEFAULT NULL COMMENT '源站小说ID',
+  `cat_id` int(11) DEFAULT NULL COMMENT '分类ID',
+  `book_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说名',
+  `author_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说作者名',
+  `task_status` tinyint(1) DEFAULT '2' COMMENT '任务状态，0：失败，1：成功，2；未执行',
+  `exc_count` tinyint(2) DEFAULT '0' COMMENT '已经执行次数，最多执行5次',
+  `create_time` datetime DEFAULT NULL COMMENT '创建时间',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='抓取单本小说任务表';
+
+-- ----------------------------
+-- Records of crawl_single_task
+-- ----------------------------
+INSERT INTO `crawl_single_task` VALUES ('6', '2', '百书斋', '1', '1', '1', '1', '0', '5', '2020-06-15 14:36:07');
+INSERT INTO `crawl_single_task` VALUES ('7', '5', '笔趣阁', '108_108291', '1', '衍天志之不朽仙', '白衣少年丶', '1', '1', '2020-06-15 14:46:08');

sql/novel_plus.sql

@@ -1691,4 +1691,144 @@ INSERT INTO `user_read_history` VALUES ('113', '1255060328322027520', '125467697
 INSERT INTO `user_read_history` VALUES ('117', '1255060328322027520', '1254946661743603712', '1254946914001629184', '2020-04-30 19:37:09', '2020-04-30 19:37:09');
 INSERT INTO `user_read_history` VALUES ('118', '1255060328322027520', '1254957312633352192', '3335449', '2020-04-30 19:37:36', '2020-04-30 19:37:36');
 
+
+
+DROP TABLE IF EXISTS `book_content0`;
+CREATE TABLE `book_content0` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1155 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content1
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content1`;
+CREATE TABLE `book_content1` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=406 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content2
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content2`;
+CREATE TABLE `book_content2` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1222 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content3
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content3`;
+CREATE TABLE `book_content3` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=410 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content4
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content4`;
+CREATE TABLE `book_content4` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1188 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content5
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content5`;
+CREATE TABLE `book_content5` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=416 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content6
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content6`;
+CREATE TABLE `book_content6` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1180 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content7
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content7`;
+CREATE TABLE `book_content7` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=404 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content8
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content8`;
+CREATE TABLE `book_content8` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1134 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content9
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content9`;
+CREATE TABLE `book_content9` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+
+DROP TABLE IF EXISTS `crawl_single_task`;
+CREATE TABLE `crawl_single_task` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `source_id` int(11) DEFAULT NULL COMMENT '爬虫源ID',
+  `source_name` varchar(50) DEFAULT NULL COMMENT '爬虫源名',
+  `source_book_id` varchar(255) DEFAULT NULL COMMENT '源站小说ID',
+  `cat_id` int(11) DEFAULT NULL COMMENT '分类ID',
+  `book_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说名',
+  `author_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说作者名',
+  `task_status` tinyint(1) DEFAULT '2' COMMENT '任务状态，0：失败，1：成功，2；未执行',
+  `exc_count` tinyint(2) DEFAULT '0' COMMENT '已经执行次数，最多执行5次',
+  `create_time` datetime DEFAULT NULL COMMENT '创建时间',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='抓取单本小说任务表';
+
+-- ----------------------------
+-- Records of crawl_single_task
+-- ----------------------------
+INSERT INTO `crawl_single_task` VALUES ('6', '2', '百书斋', '1', '1', '1', '1', '0', '5', '2020-06-15 14:36:07');
+INSERT INTO `crawl_single_task` VALUES ('7', '5', '笔趣阁', '108_108291', '1', '衍天志之不朽仙', '白衣少年丶', '1', '1', '2020-06-15 14:46:08');
 UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n	\"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n	\"catIdRule\": {\n		\"catId1\": \"1\",\n		\"catId2\": \"2\",\n		\"catId3\": \"3\",\n		\"catId4\": \"4\",\n		\"catId5\": \"7\",\n		\"catId6\": \"6\",\n		\"catId7\": \"8\"\n	},\n	\"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n	\"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n	\"totalPagePatten\": \"第\\\\d+/(\\\\d+)页\",\n	\"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n	\"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n	\"authorNamePatten\": \"<p>作者：([^/]+)</p>\",\n	\"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n	\"statusPatten\": \"<p>状态：([^/]+)</p>\",\n	\"bookStatusRule\": {\n		\"连载中\": 0,\n		\"完本\": 1\n	},\n	\"descStart\": \"<div class=\\\"intro_info\\\">\",\n	\"descEnd\": \"最新章节推荐地址\",\n	\"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n	\"bookIndexStart\": \"<dt>《\",\n	\"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n	\"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n	\"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n	\"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n	\"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;