Update README.md

update

README.md

@@ -1,7 +1,15 @@
-[![index](./assets/热门云产品1040.100.jpg)](https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console)
+[![index]( https://s1.ax1x.com/2020/07/03/NOSA5q.jpg )]( https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console )
 
 # 小说精品屋-plus
 
+#### 新项目：小说精品屋-微服务版
+
+基于小说精品屋-plus构建的Spring Cloud 微服务小说门户平台。
+
+Gitee仓库地址： https://gitee.com/xiongxyang/novel-cloud
+
+GitHub仓库地址： https://github.com/201206030/novel-cloud
+
 #### 演示地址
 
 [点击前往](http://www.java2nb.com)（前台门户）
@@ -40,7 +48,31 @@ novel-plus -- 父工程
 ```
 
 #### 技术选型
-Springboot+Mybatis+Mysql+Ehcache+Thymeleaf+Layui
+
+| 技术                 | 说明                                                         
+| -------------------- | ---------------------------
+| SpringBoot           | Spring应用快速开发脚手架     
+| MyBatis              | 持久层ORM框架 
+| MyBatis Dynamic SQL  | Mybatis动态sql
+| PageHelper           | MyBatis分页插件
+| MyBatisGenerator     | 持久层代码生成插件
+| Sharding-Jdbc        | 代码层分库分表中间件
+| JJWT                 | JWT登录支持  
+| SpringSecurity       | 安全框架                           
+| Shiro                | 安全框架  
+| Ehcache              | Java进程内缓存框架(默认缓存)  
+| Redis                | 分布式缓存(缓存替换方案，默认关闭，一行配置开启)                               
+| ElasticSearch        | 搜索引擎(搜索增强方案，默认关闭，一行配置开启)                      
+| RabbitMq             | 消息队列(流量削峰，默认关闭，一行配置开启)  
+| OSS                  | 阿里云对象存储服务(图片存储方式之一，一行配置即可切换) 
+| FastDfs              |开源轻量级分布式文件系统(图片存储方式之一，一行配置即可切换)                      
+| Redisson             | 实现分布式锁                                       
+| Lombok               | 简化对象封装工具                                                                               
+| Docker               | 应用容器引擎   
+| Mysql                | 数据库服务   
+| Thymeleaf            | 模板引擎     
+| Layui                | 前端UI                    
+                 
 
 #### PC站截图
 
@@ -52,69 +84,67 @@ Springboot+Mybatis+Mysql+Ehcache+Thymeleaf+Layui
 
 ![img](https://oscimg.oschina.net/oscnet/up-d0b2e03129bfae47b8bb96a491b73d383c5.png)
 
-3. 排行榜
+3. 搜索页
+
+![img](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/QQ20200520-215756.png)
+
+![img](https://oscimg.oschina.net/oscnet/up-ed5f689557718924acac76bc3ebca36afcb.png)
+
+4. 排行榜
 
 ![img](https://oscimg.oschina.net/oscnet/up-78d5a68586cd92a86c669311f414508f922.png)
 
-4. 详情页
+5. 详情页
 
 ![img](https://oscimg.oschina.net/oscnet/up-8be2495a2869f93626b0c9c1df6f329747a.png)
 
-5. 阅读页
+6. 阅读页
 
 ![img](https://oscimg.oschina.net/oscnet/up-517c84148d2db8e11717a8bbecc57fa1be7.png)
 
-6. 用户中心
+7. 用户中心
 
 ![img](https://oscimg.oschina.net/oscnet/up-805a30e7a663a3fd5cb39a7ea26bc132a01.png)
 
-7. 充值
+8. 充值
 
 ![img](https://oscimg.oschina.net/oscnet/up-5a601b0b3af3224d0bebcfe12fc15075d34.png)
 
 ![img](https://oscimg.oschina.net/oscnet/up-face25d02c07b05b2ce954cc4bf4ee6a0cc.png)
 
-8.作家专区
+9. 作家专区
 
 ![img](https://oscimg.oschina.net/oscnet/up-30766372cc7f56480ff1d7d55198204f6ea.png)
 
 ![img](https://oscimg.oschina.net/oscnet/up-9737995e465b86f3bee3211221f6c3b8a56.png)
 
-9.购买
+10. 购买
 
 ![img](https://oscimg.oschina.net/oscnet/up-ce0f585efd82a9670335f118cf5897c85e9.png)
 
 ![img](https://oscimg.oschina.net/oscnet/up-f849960f4c1303fea77d26e64fc505a7180.png)
 
-
-
 #### 手机站截图
 
 1. 首页
 
-   ![index](./assets/QQ%E5%9B%BE%E7%89%8720191018162208.jpg)
-
-   
+   ![index](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/QQ%E5%9B%BE%E7%89%8720191018162208.jpg)
 
 2. 小说详情页
 
-   ![微信图片_20190904181558](./assets/%E5%BE%AE%E4%BF%A1%E5%9B%BE%E7%89%87_20190904181558.png)
-
-   
+   ![微信图片_20190904181558](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/%E5%BE%AE%E4%BF%A1%E5%9B%BE%E7%89%87_20190904181558.png)
 
 3. 目录页
 
-   ![QQ图片20191018161901](./assets/QQ%E5%9B%BE%E7%89%8720191108022250.png)
+   ![QQ图片20191018161901](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/QQ%E5%9B%BE%E7%89%8720191108022250.png)
 
 4. 小说阅读页
 
-   ![QQ图片20191018161901](./assets/QQ%E5%9B%BE%E7%89%8720191018161901.png)
-
-   
+   ![QQ图片20191018161901](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/QQ%E5%9B%BE%E7%89%8720191018161901.png)
 
 #### 爬虫管理系统截图
 
-![QQ图片20191018161901](./assets/crawl_index.png)
+![img](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.2/assets/crawl_index.png)
 
 #### 后台管理系统截图
 
@@ -161,16 +191,16 @@ docker安装教程：[点击前往](https://my.oschina.net/java2nb/blog/4271989)
 
 #### QQ交流群
 
-![mini-code](./assets/小说精品屋开源项目交流群群聊二维码.png)
+![mini-code](https://s1.ax1x.com/2020/10/31/BUQVeI.png)
 
 #### 捐赠支持
 
 开源项目不易，若此项目能得到你的青睐，可以捐赠支持作者持续开发与维护。 
 
-![mini-code](./assets/jk.png)
+![mini-code](https://s1.ax1x.com/2020/10/31/BUQJwq.png)
 
 #### 备注
 
 精品小说屋所有相关项目均已在开源中国公开，感兴趣的可进入[开源中国](https://www.oschina.net/p/fiction_house)按关键字`精品小说屋`搜索。
 
-[![index](./assets/120060.jpg)](https://www.aliyun.com/minisite/goods?userCode=uf4nasee )
+[![index](https://s1.ax1x.com/2020/07/03/NOSuMF.jpg)](https://www.aliyun.com/minisite/goods?userCode=uf4nasee )

es/index_create.txt

@@ -0,0 +1,82 @@
+PUT /novel
+{
+  "mappings" : {
+    "book" : {
+      "properties" : {
+        "id" : {
+          "type" : "long"
+        },
+        "authorId" : {
+          "type" : "long"
+        },
+        "authorName" : {
+          "type" : "text",
+          "analyzer": "ik_smart",
+          "boost": 1.9
+        },
+
+        "bookName" : {
+          "type" : "text",
+          "analyzer": "ik_smart",
+          "boost": 2
+        },
+
+        "bookDesc" : {
+          "type" : "text",
+          "analyzer": "ik_smart",
+          "boost": 0.1
+        },
+
+        "bookStatus" : {
+          "type" : "short"
+        },
+
+        "catId" : {
+          "type" : "integer"
+        },
+
+        "catName" : {
+          "type" : "text",
+          "analyzer": "ik_smart",
+          "boost": 1.0
+        },
+
+        "lastIndexId" : {
+          "type" : "long"
+        },
+
+        "lastIndexName" : {
+          "type" : "text",
+          "analyzer": "ik_smart",
+          "boost": 0.1
+        },
+
+        "lastIndexUpdateTime" : {
+          "type": "keyword"
+        },
+
+        "picUrl" : {
+          "type" : "keyword"
+        },
+
+        "score" : {
+          "type" : "float"
+        },
+
+        "wordCount" : {
+          "type" : "integer"
+        },
+
+        "workDirection" : {
+          "type" : "short"
+        },
+
+        "visitCount" : {
+          "type": "long"
+        }
+
+
+      }
+    }
+  }
+}

novel-admin/Dockerfile

@@ -0,0 +1,6 @@
+FROM java:8
+ADD novel-admin-1.0.0.jar /root
+ENV dburl=""
+ENV username=""
+ENV	password=""
+ENTRYPOINT ["sh","-c","java -Dspring.datasource.url=${dburl} -Dspring.datasource.username=${username} -Dspring.datasource.password=${password} -jar /root/novel-admin-1.0.0.jar"]

novel-admin/logs/debug.log

@@ -1,8 +0,0 @@
-2020-05-13 21:52:00,972 INFO (StartupInfoLogger.java:50)- Starting TestDemo on USER-20180729KA with PID 3532 (started by Administrator in E:\baseprojectparent\novel-plus\novel-admin)
-2020-05-13 21:52:01,113 DEBUG (StartupInfoLogger.java:53)- Running with Spring Boot v2.0.1.RELEASE, Spring v5.0.5.RELEASE
-2020-05-13 21:52:01,131 INFO (SpringApplication.java:663)- The following profiles are active: dev
-2020-05-13 21:52:54,469 DEBUG (ApplicationContextRegister.java:29)- ApplicationContext registed-->org.springframework.web.context.support.GenericWebApplicationContext@5b529706: startup date [Wed May 13 21:52:01 CST 2020]; root of context hierarchy
-2020-05-13 21:53:49,622 INFO (StartupInfoLogger.java:59)- Started TestDemo in 114.268 seconds (JVM running for 124.957)
-2020-05-18 09:48:03,219 INFO (StartupInfoLogger.java:50)- Starting TestDemo on DESKTOP-CPCLUI6 with PID 13172 (started by 11797 in D:\gitee\novel-plus\novel-admin)
-2020-05-18 09:48:03,223 DEBUG (StartupInfoLogger.java:53)- Running with Spring Boot v2.0.1.RELEASE, Spring v5.0.5.RELEASE
-2020-05-18 09:48:03,227 INFO (SpringApplication.java:663)- The following profiles are active: dev

novel-admin/pom.xml

@@ -137,10 +137,6 @@
             <artifactId>velocity</artifactId>
             <version>1.7</version>
         </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-cache</artifactId>
-        </dependency>
         <!--<dependency>-->
         <!--<groupId>org.springframework.boot</groupId>-->
         <!--<artifactId>spring-boot-starter-data-redis</artifactId>-->

novel-admin/src/main/resources/application-dev.yml

@@ -12,7 +12,7 @@ spring:
     driverClassName: com.mysql.jdbc.Driver
     url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai
     username: root
-    password:
+    password: test123456
     #password:
     initialSize: 1
     minIdle: 3

novel-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.0.2</version>
+        <version>2.6.4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -31,11 +31,20 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-cache</artifactId>
         </dependency>
+
+        <!--ehcache-->
         <dependency>
             <groupId>net.sf.ehcache</groupId>
             <artifactId>ehcache</artifactId>
         </dependency>
 
+        <!--集成redis-->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-redis</artifactId>
+            <version>${redis.version}</version>
+        </dependency>
+
         <!--mysql驱动-->
         <dependency>
             <groupId>mysql</groupId>
@@ -48,6 +57,22 @@
             <version>${mybatis.version}</version>
         </dependency>
 
+        <!-- 分库分表-->
+        <!-- sharding jdbc依赖 -->
+
+        <dependency>
+            <groupId>io.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-spring-boot-starter</artifactId>
+            <version>${sharding.jdbc.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-spring-namespace</artifactId>
+            <version>${sharding.jdbc.version}</version>
+        </dependency>
+
+
         <!--分页助手启动器-->
         <dependency>
             <groupId>com.github.pagehelper</groupId>

novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java

@@ -46,4 +46,23 @@ public interface CacheKey {
      * */
     String RUNNING_CRAWL_THREAD_KEY_PREFIX = "runningCrawlTreadDataKeyPrefix";
 
+    /**
+     * 上一次搜索引擎更新的时间
+     * */
+    String ES_LAST_UPDATE_TIME = "esLastUpdateTime";
+
+    /**
+     * 搜索引擎转换锁
+     * */
+    String ES_TRANS_LOCK = "esTransLock";
+
+    /**
+     * 上一次搜索引擎是否更新过小说点击量
+     * */
+    String ES_IS_UPDATE_VISIT = "esIsUpdateVisit";
+
+    /**
+     * 累积的小说点击量
+     * */
+    String BOOK_ADD_VISIT_COUNT = "bookAddVisitCount";
 }

novel-common/src/main/java/com/java2nb/novel/core/cache/CacheService.java

@@ -51,9 +51,5 @@ public interface CacheService {
 	 * */
 	void expire(String key, long timeout);
 
-	/**
-	 * 刷新缓存
-	 * */
-	void refresh(String key);
 
 }

novel-common/src/main/java/com/java2nb/novel/core/cache/impl/EhCacheServiceImpl.java

@@ -1,24 +1,25 @@
 package com.java2nb.novel.core.cache.impl;
 
 import com.java2nb.novel.core.cache.CacheService;
+import lombok.RequiredArgsConstructor;
 import net.sf.ehcache.Cache;
 import net.sf.ehcache.CacheManager;
 import net.sf.ehcache.Element;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Service;
 
 /**
  * @author xxy
  */
+@ConditionalOnProperty(prefix = "cache", name = "type", havingValue = "ehcache")
+@RequiredArgsConstructor
 @Service
 public class EhCacheServiceImpl implements CacheService {
 	
-	@Autowired
-	private CacheManager cacheManager ;
+	private final CacheManager cacheManager ;
  
-	private static final String CACHE_NAME = "utilCache";
- 
-	
+
 	/**
 	 * 获得一个Cache，没有则创建一个。
 	 * @return
@@ -30,14 +31,6 @@ public class EhCacheServiceImpl implements CacheService {
 	}
 	
  
-	public  CacheManager getCacheManager() {
-		return cacheManager;
-	}
-	
-	
-
-
-
 	@Override
 	public String get(String key) {
 		Element element = getCache().get(key);
@@ -125,20 +118,6 @@ public class EhCacheServiceImpl implements CacheService {
 	}
 
 
-	@Override
-	public void refresh(String key) {
-		Element element = getCache().get(key);
-		if (element != null) {
-			Object value = element.getValue();
-			int timeToLive = element.getTimeToLive();
-			element = new Element(key, value);
-			element.setTimeToLive(timeToLive);
-			Cache cache = getCache();
-			cache.put(element);
-		}
-		
-	}
-
 
 
 }

novel-common/src/main/java/com/java2nb/novel/core/cache/impl/RedisServiceImpl.java

@@ -0,0 +1,73 @@
+package com.java2nb.novel.core.cache.impl;
+
+import com.java2nb.novel.core.cache.CacheService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Service;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author xxy
+ */
+@ConditionalOnProperty(prefix = "cache", name = "type", havingValue = "redis")
+@RequiredArgsConstructor
+@Service
+public class RedisServiceImpl implements CacheService {
+
+	private final StringRedisTemplate stringRedisTemplate;
+
+	private final RedisTemplate<Object,  Object>  redisTemplate;
+
+
+	@Override
+	public String get(String key) {
+		return stringRedisTemplate.opsForValue().get(key);
+	}
+
+	@Override
+	public void set(String key, String value) {
+		stringRedisTemplate.opsForValue().set(key,value);
+	}
+
+	@Override
+	public void set(String key, String value, long timeout) {
+		stringRedisTemplate.opsForValue().set(key,value,timeout, TimeUnit.SECONDS);
+
+	}
+
+	@Override
+	public Object getObject(String key) {
+		return redisTemplate.opsForValue().get(key);
+	}
+
+	@Override
+	public void setObject(String key, Object value) {
+		redisTemplate.opsForValue().set(key,value);
+	}
+
+	@Override
+	public void setObject(String key, Object value, long timeout) {
+		redisTemplate.opsForValue().set(key,value,timeout, TimeUnit.SECONDS);
+	}
+
+	@Override
+	public void del(String key) {
+		redisTemplate.delete(key);
+		stringRedisTemplate.delete(key);
+	}
+
+	@Override
+	public boolean contains(String key) {
+		return redisTemplate.hasKey(key) || stringRedisTemplate.hasKey(key);
+	}
+
+	@Override
+	public void expire(String key, long timeout) {
+		redisTemplate.expire(key,timeout, TimeUnit.SECONDS);
+		stringRedisTemplate.expire(key,timeout, TimeUnit.SECONDS);
+	}
+
+}

novel-common/src/main/java/com/java2nb/novel/core/enums/ResponseStatus.java

@@ -55,9 +55,18 @@ public enum ResponseStatus {
      * */
     INVITE_CODE_INVALID(4001, "邀请码无效！"),
     AUTHOR_STATUS_FORBIDDEN(4002, "作者状态异常，暂不能管理小说！")
-    , BOOKNAME_EXISTS(4003,"已发布过同名小说！")
+    , BOOKNAME_EXISTS(4003,"已发布过同名小说！"),
 
-    ,
+    /**
+     * 小说相关错误
+     */
+    BOOK_EXISTS(5001,"该小说已存在")
+
+            ,
+    /**
+     * 搜索引擎相关错误
+     * */
+    ES_SEARCH_FAIL(9001,"搜索引擎查询错误！"),
 
 
     /**

novel-common/src/main/java/com/java2nb/novel/core/utils/FileUtil.java

@@ -1,6 +1,7 @@
 package com.java2nb.novel.core.utils;
 
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.Charsets;
 import org.apache.http.client.utils.DateUtils;
 import org.springframework.core.io.Resource;
@@ -9,6 +10,7 @@ import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
 
+import javax.imageio.ImageIO;
 import java.io.*;
 import java.util.Date;
 import java.util.Objects;
@@ -17,34 +19,65 @@ import java.util.Objects;
  * 文件操作工具类
  * @author 11797
  */
+@Slf4j
 public class FileUtil {
 
     /**
      * 网络图片转本地
      * */
-    @SneakyThrows
     public static String network2Local(String picSrc,String picSavePath,String visitPrefix) {
-        //本地图片保存
-        HttpHeaders headers = new HttpHeaders();
-        HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
-        ResponseEntity<Resource> resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1.name()).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
-        InputStream input = Objects.requireNonNull(resEntity.getBody()).getInputStream();
-        Date currentDate = new Date();
-        picSrc = visitPrefix  + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd") + "/"
-                + UUIDUtil.getUUID32()
-                + picSrc.substring(picSrc.lastIndexOf("."));
-        File picFile = new File(picSavePath + picSrc);
-        File parentFile = picFile.getParentFile();
-        if (!parentFile.exists()) {
-            parentFile.mkdirs();
+        InputStream input = null;
+        OutputStream out = null;
+        try {
+            //本地图片保存
+            HttpHeaders headers = new HttpHeaders();
+            HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
+            ResponseEntity<Resource> resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1.name()).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
+            input = Objects.requireNonNull(resEntity.getBody()).getInputStream();
+            Date currentDate = new Date();
+            picSrc = visitPrefix + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd") + "/"
+                    + UUIDUtil.getUUID32()
+                    + picSrc.substring(picSrc.lastIndexOf("."));
+            File picFile = new File(picSavePath + picSrc);
+            File parentFile = picFile.getParentFile();
+            if (!parentFile.exists()) {
+                parentFile.mkdirs();
+            }
+            out = new FileOutputStream(picFile);
+            byte[] b = new byte[4096];
+            for (int n; (n = input.read(b)) != -1; ) {
+                out.write(b, 0, n);
+            }
+
+            out.flush();
+            if( ImageIO.read(picFile) == null){
+                picSrc = "/images/default.gif";
+            }
+
+        }catch (Exception e){
+            log.error(e.getMessage(),e);
+
+            picSrc = "/images/default.gif";
+        }finally {
+            if(input != null){
+                try {
+                    input.close();
+                } catch (IOException e) {
+                    log.error(e.getMessage(),e);
+                }finally {
+                    if(out != null){
+                        try {
+                            out.close();
+                        } catch (IOException e) {
+                            log.error(e.getMessage(),e);
+                        }
+                    }
+                }
+            }
         }
-        OutputStream out = new FileOutputStream(picFile);
-        byte[] b = new byte[4096];
-        for (int n; (n = input.read(b)) != -1; ) {
-            out.write(b, 0, n);
-        }
-        out.close();
-        input.close();
+
+
+
         return picSrc;
     }
 

novel-common/src/main/java/com/java2nb/novel/core/utils/StringUtil.java

@@ -0,0 +1,73 @@
+package com.java2nb.novel.core.utils;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Arrays;
+
+/**
+ * @author xiongxiaoyang
+ */
+public class StringUtil {
+
+    /**
+     * 将驼峰式命名的字符串转换为下划线大写方式。如果转换前的驼峰式命名的字符串为空，则返回空字符串。</br>
+     * 例如：HelloWorld->HELLO_WORLD
+     * @param name 转换前的驼峰式命名的字符串
+     * @return 转换后下划线大写方式命名的字符串
+     */
+    public static String underscoreName(String name) {
+        StringBuilder result = new StringBuilder();
+        if (name != null && name.length() > 0) {
+            // 将第一个字符处理成大写
+            result.append(name.substring(0, 1).toUpperCase());
+            // 循环处理其余字符
+            for (int i = 1; i < name.length(); i++) {
+                String s = name.substring(i, i + 1);
+                // 在大写字母前添加下划线
+                if (s.equals(s.toUpperCase()) && !Character.isDigit(s.charAt(0))) {
+                    result.append("_");
+                }
+                // 其他字符直接转成大写
+                result.append(s.toUpperCase());
+            }
+        }
+        return result.toString();
+    }
+
+    /**
+     * 将下划线大写方式命名的字符串转换为驼峰式。如果转换前的下划线大写方式命名的字符串为空，则返回空字符串。</br>
+     * 例如：HELLO_WORLD->HelloWorld
+     * @param name 转换前的下划线大写方式命名的字符串
+     * @return 转换后的驼峰式命名的字符串
+     */
+    public static String camelName(String name) {
+        StringBuilder result = new StringBuilder();
+        // 快速检查
+        if (name == null || name.isEmpty()) {
+            // 没必要转换
+            return "";
+        } else if (!name.contains("_")) {
+            // 不含下划线，仅将首字母小写
+            return name.substring(0, 1).toLowerCase() + name.substring(1);
+        }
+        // 用下划线将原始字符串分割
+        String camels[] = name.split("_");
+        for (String camel : camels) {
+            // 跳过原始字符串中开头、结尾的下换线或双重下划线
+            if (camel.isEmpty()) {
+                continue;
+            }
+            // 处理真正的驼峰片段
+            if (result.length() == 0) {
+                // 第一个驼峰片段，全部字母都小写
+                result.append(camel.toLowerCase());
+            } else {
+                // 其他的驼峰片段，首字母大写
+                result.append(camel.substring(0, 1).toUpperCase());
+                result.append(camel.substring(1).toLowerCase());
+            }
+        }
+        return result.toString();
+    }
+
+}

novel-common/src/main/java/com/java2nb/novel/entity/CrawlSingleTask.java

@@ -10,9 +10,21 @@ public class CrawlSingleTask {
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     private Integer sourceId;
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    private String sourceName;
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     private String sourceBookId;
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    private Integer catId;
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    private String bookName;
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    private String authorName;
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     private Byte taskStatus;
 
@@ -42,6 +54,16 @@ public class CrawlSingleTask {
         this.sourceId = sourceId;
     }
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public String getSourceName() {
+        return sourceName;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public void setSourceName(String sourceName) {
+        this.sourceName = sourceName == null ? null : sourceName.trim();
+    }
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public String getSourceBookId() {
         return sourceBookId;
@@ -52,6 +74,36 @@ public class CrawlSingleTask {
         this.sourceBookId = sourceBookId == null ? null : sourceBookId.trim();
     }
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public Integer getCatId() {
+        return catId;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public void setCatId(Integer catId) {
+        this.catId = catId;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public String getBookName() {
+        return bookName;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public void setBookName(String bookName) {
+        this.bookName = bookName == null ? null : bookName.trim();
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public String getAuthorName() {
+        return authorName;
+    }
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public void setAuthorName(String authorName) {
+        this.authorName = authorName == null ? null : authorName.trim();
+    }
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public Byte getTaskStatus() {
         return taskStatus;

novel-common/src/main/java/com/java2nb/novel/mapper/BookContentDynamicSqlSupport.java

@@ -24,7 +24,7 @@ public final class BookContentDynamicSqlSupport {
 
         public final SqlColumn<Long> indexId = column("index_id", JDBCType.BIGINT);
 
-        public final SqlColumn<String> content = column("content", JDBCType.LONGVARCHAR);
+        public final SqlColumn<String> content = column("content", JDBCType.VARCHAR);
 
         public BookContent() {
             super("book_content");

novel-common/src/main/java/com/java2nb/novel/mapper/BookContentMapper.java

@@ -63,7 +63,7 @@ public interface BookContentMapper {
     @Results(id="BookContentResult", value = {
         @Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
         @Result(column="index_id", property="indexId", jdbcType=JdbcType.BIGINT),
-        @Result(column="content", property="content", jdbcType=JdbcType.LONGVARCHAR)
+        @Result(column="content", property="content", jdbcType=JdbcType.VARCHAR)
     })
     List<BookContent> selectMany(SelectStatementProvider selectStatement);
 

novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskDynamicSqlSupport.java

@@ -16,9 +16,21 @@ public final class CrawlSingleTaskDynamicSqlSupport {
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public static final SqlColumn<Integer> sourceId = crawlSingleTask.sourceId;
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public static final SqlColumn<String> sourceName = crawlSingleTask.sourceName;
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public static final SqlColumn<String> sourceBookId = crawlSingleTask.sourceBookId;
 
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public static final SqlColumn<Integer> catId = crawlSingleTask.catId;
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public static final SqlColumn<String> bookName = crawlSingleTask.bookName;
+
+    @Generated("org.mybatis.generator.api.MyBatisGenerator")
+    public static final SqlColumn<String> authorName = crawlSingleTask.authorName;
+
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     public static final SqlColumn<Byte> taskStatus = crawlSingleTask.taskStatus;
 
@@ -34,8 +46,16 @@ public final class CrawlSingleTaskDynamicSqlSupport {
 
         public final SqlColumn<Integer> sourceId = column("source_id", JDBCType.INTEGER);
 
+        public final SqlColumn<String> sourceName = column("source_name", JDBCType.VARCHAR);
+
         public final SqlColumn<String> sourceBookId = column("source_book_id", JDBCType.VARCHAR);
 
+        public final SqlColumn<Integer> catId = column("cat_id", JDBCType.INTEGER);
+
+        public final SqlColumn<String> bookName = column("book_name", JDBCType.VARCHAR);
+
+        public final SqlColumn<String> authorName = column("author_name", JDBCType.VARCHAR);
+
         public final SqlColumn<Byte> taskStatus = column("task_status", JDBCType.TINYINT);
 
         public final SqlColumn<Byte> excCount = column("exc_count", JDBCType.TINYINT);

novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskMapper.java

@@ -35,7 +35,7 @@ import org.mybatis.dynamic.sql.util.mybatis3.MyBatis3Utils;
 @Mapper
 public interface CrawlSingleTaskMapper {
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
-    BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceBookId, taskStatus, excCount, createTime);
+    BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceName, sourceBookId, catId, bookName, authorName, taskStatus, excCount, createTime);
 
     @Generated("org.mybatis.generator.api.MyBatisGenerator")
     @SelectProvider(type=SqlProviderAdapter.class, method="select")
@@ -63,7 +63,11 @@ public interface CrawlSingleTaskMapper {
     @Results(id="CrawlSingleTaskResult", value = {
         @Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
         @Result(column="source_id", property="sourceId", jdbcType=JdbcType.INTEGER),
+        @Result(column="source_name", property="sourceName", jdbcType=JdbcType.VARCHAR),
         @Result(column="source_book_id", property="sourceBookId", jdbcType=JdbcType.VARCHAR),
+        @Result(column="cat_id", property="catId", jdbcType=JdbcType.INTEGER),
+        @Result(column="book_name", property="bookName", jdbcType=JdbcType.VARCHAR),
+        @Result(column="author_name", property="authorName", jdbcType=JdbcType.VARCHAR),
         @Result(column="task_status", property="taskStatus", jdbcType=JdbcType.TINYINT),
         @Result(column="exc_count", property="excCount", jdbcType=JdbcType.TINYINT),
         @Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP)
@@ -96,7 +100,11 @@ public interface CrawlSingleTaskMapper {
         return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c ->
             c.map(id).toProperty("id")
             .map(sourceId).toProperty("sourceId")
+            .map(sourceName).toProperty("sourceName")
             .map(sourceBookId).toProperty("sourceBookId")
+            .map(catId).toProperty("catId")
+            .map(bookName).toProperty("bookName")
+            .map(authorName).toProperty("authorName")
             .map(taskStatus).toProperty("taskStatus")
             .map(excCount).toProperty("excCount")
             .map(createTime).toProperty("createTime")
@@ -108,7 +116,11 @@ public interface CrawlSingleTaskMapper {
         return MyBatis3Utils.insertMultiple(this::insertMultiple, records, crawlSingleTask, c ->
             c.map(id).toProperty("id")
             .map(sourceId).toProperty("sourceId")
+            .map(sourceName).toProperty("sourceName")
             .map(sourceBookId).toProperty("sourceBookId")
+            .map(catId).toProperty("catId")
+            .map(bookName).toProperty("bookName")
+            .map(authorName).toProperty("authorName")
             .map(taskStatus).toProperty("taskStatus")
             .map(excCount).toProperty("excCount")
             .map(createTime).toProperty("createTime")
@@ -120,7 +132,11 @@ public interface CrawlSingleTaskMapper {
         return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c ->
             c.map(id).toPropertyWhenPresent("id", record::getId)
             .map(sourceId).toPropertyWhenPresent("sourceId", record::getSourceId)
+            .map(sourceName).toPropertyWhenPresent("sourceName", record::getSourceName)
             .map(sourceBookId).toPropertyWhenPresent("sourceBookId", record::getSourceBookId)
+            .map(catId).toPropertyWhenPresent("catId", record::getCatId)
+            .map(bookName).toPropertyWhenPresent("bookName", record::getBookName)
+            .map(authorName).toPropertyWhenPresent("authorName", record::getAuthorName)
             .map(taskStatus).toPropertyWhenPresent("taskStatus", record::getTaskStatus)
             .map(excCount).toPropertyWhenPresent("excCount", record::getExcCount)
             .map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime)
@@ -158,7 +174,11 @@ public interface CrawlSingleTaskMapper {
     static UpdateDSL<UpdateModel> updateAllColumns(CrawlSingleTask record, UpdateDSL<UpdateModel> dsl) {
         return dsl.set(id).equalTo(record::getId)
                 .set(sourceId).equalTo(record::getSourceId)
+                .set(sourceName).equalTo(record::getSourceName)
                 .set(sourceBookId).equalTo(record::getSourceBookId)
+                .set(catId).equalTo(record::getCatId)
+                .set(bookName).equalTo(record::getBookName)
+                .set(authorName).equalTo(record::getAuthorName)
                 .set(taskStatus).equalTo(record::getTaskStatus)
                 .set(excCount).equalTo(record::getExcCount)
                 .set(createTime).equalTo(record::getCreateTime);
@@ -168,7 +188,11 @@ public interface CrawlSingleTaskMapper {
     static UpdateDSL<UpdateModel> updateSelectiveColumns(CrawlSingleTask record, UpdateDSL<UpdateModel> dsl) {
         return dsl.set(id).equalToWhenPresent(record::getId)
                 .set(sourceId).equalToWhenPresent(record::getSourceId)
+                .set(sourceName).equalToWhenPresent(record::getSourceName)
                 .set(sourceBookId).equalToWhenPresent(record::getSourceBookId)
+                .set(catId).equalToWhenPresent(record::getCatId)
+                .set(bookName).equalToWhenPresent(record::getBookName)
+                .set(authorName).equalToWhenPresent(record::getAuthorName)
                 .set(taskStatus).equalToWhenPresent(record::getTaskStatus)
                 .set(excCount).equalToWhenPresent(record::getExcCount)
                 .set(createTime).equalToWhenPresent(record::getCreateTime);
@@ -178,7 +202,11 @@ public interface CrawlSingleTaskMapper {
     default int updateByPrimaryKey(CrawlSingleTask record) {
         return update(c ->
             c.set(sourceId).equalTo(record::getSourceId)
+            .set(sourceName).equalTo(record::getSourceName)
             .set(sourceBookId).equalTo(record::getSourceBookId)
+            .set(catId).equalTo(record::getCatId)
+            .set(bookName).equalTo(record::getBookName)
+            .set(authorName).equalTo(record::getAuthorName)
             .set(taskStatus).equalTo(record::getTaskStatus)
             .set(excCount).equalTo(record::getExcCount)
             .set(createTime).equalTo(record::getCreateTime)
@@ -190,7 +218,11 @@ public interface CrawlSingleTaskMapper {
     default int updateByPrimaryKeySelective(CrawlSingleTask record) {
         return update(c ->
             c.set(sourceId).equalToWhenPresent(record::getSourceId)
+            .set(sourceName).equalToWhenPresent(record::getSourceName)
             .set(sourceBookId).equalToWhenPresent(record::getSourceBookId)
+            .set(catId).equalToWhenPresent(record::getCatId)
+            .set(bookName).equalToWhenPresent(record::getBookName)
+            .set(authorName).equalToWhenPresent(record::getAuthorName)
             .set(taskStatus).equalToWhenPresent(record::getTaskStatus)
             .set(excCount).equalToWhenPresent(record::getExcCount)
             .set(createTime).equalToWhenPresent(record::getCreateTime)

novel-common/src/main/resources/application-common-dev.yml

@@ -1,19 +1,77 @@
 spring:
   profiles:
     include: [common]
+  main:
+    allow-bean-definition-overriding: true
+  #Redis服务器IP
+  redis:
+    host: 127.0.0.1
+    #Redis服务器连接端口
+    port: 6379
+    #Redis服务器连接密码
+    password: test
+    jedis:
+      pool:
+        #连接池最大连接数（使用负值表示没有限制）
+        max-active: 8
+        #连接池最大阻塞等待时间（使用负值表示没有限制）
+        max-wait: 1
+        #连接池最大阻塞等待时间（使用负值表示没有限制）
+        max-idle: 8
+        #连接池中的最小空闲连接
+        min-idle: 0
+    #连接超时时间（毫秒）
+    timeout: 30000
   datasource:
     url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
     username: root
     password: test123456
     driver-class-name: com.mysql.cj.jdbc.Driver
 
+####使用shardingJdbc时，
+####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误
+##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR
+
+sharding:
+  jdbc:
+    datasource:
+      names: ds0 #,ds1
+      ds0:
+        type: com.zaxxer.hikari.HikariDataSource
+        driver-class-name: com.mysql.cj.jdbc.Driver
+        jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
+        username: root
+        password: test123456
+    #      ds1:
+    #        type: com.alibaba.druid.pool.DruidDataSource
+    #        driver-class-name: com.mysql.jdbc.Driver
+    #        url: jdbc:mysql://localhost:3306/novel_plus2
+    #        username: root
+    #        password: test123456
+    config:
+      sharding:
+        props:
+          sql.show: true
+        tables:
+          book_content:  #book_content表
+            key-generator-column-name: id  #主键
+            actual-data-nodes: ds${0}.book_content${0..9}  #数据节点
+            #            database-strategy:  #分库策略
+            #              inline:
+            #                sharding-column: book_id
+            #                algorithm-expression: ds${book_id % 10}
+            table-strategy:  #分表策略
+              inline:
+                shardingColumn: index_id
+                algorithm-expression: book_content${index_id % 10}
+
+
+
+
 
 
 pic:
   save:
-    type: 2 #图片保存方式, 1不保存，使用网络图片 ，2本地保存
-    path: /var/pic  #图片保存路径
-
-
-
-
+    type: 2 #图片保存方式, 1不保存，使用爬取的网络图片 ，2保存在自己的存储介质
+    storage: local #存储介质，local：本地，OSS：阿里云对象存储，fastDfs：分布式文件系统
+    path: /var/pic  #图片保存路径

novel-common/src/main/resources/application-common.yml

@@ -11,6 +11,9 @@ spring:
     generator:
       write-numbers-as-strings: true
 
+#缓存类型，ehcache(默认)、redis
+cache:
+  type: ehcache
 
 mybatis:
   configuration:

novel-common/src/main/resources/mybatis/generatorConfig.xml

@@ -9,7 +9,7 @@
         </commentGenerator>
         <jdbcConnection
                 connectionURL="jdbc:mysql://127.0.0.1:3306/novel_plus?tinyInt1isBit=false&amp;useUnicode=true&amp;characterEncoding=utf-8&amp;serverTimezone=Asia/Shanghai&amp;nullCatalogMeansCurrent=true"
-                driverClass="com.mysql.jdbc.Driver" password=""
+                driverClass="com.mysql.jdbc.Driver" password="test123456"
                 userId="root" />
 
         <!-- 默认false，把JDBC DECIMAL 和 NUMERIC 类型解析为 Integer， 为 true时把JDBC DECIMAL
@@ -44,7 +44,7 @@
         </javaClientGenerator>
 
         <!--生成全部表tableName设为%-->
-        <table tableName="user_buy_record" domainObjectName="UserBuyRecord"/>
+        <table tableName="crawl_single_task"/>
 
         <!-- 指定数据库表 -->
         <!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->

novel-crawl/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.0.2</version>
+        <version>2.6.4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -25,6 +25,8 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-security</artifactId>
         </dependency>
+
+
     </dependencies>
 
     <build>

novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java

@@ -3,11 +3,12 @@ package com.java2nb.novel.controller;
 import com.github.pagehelper.PageInfo;
 import com.java2nb.novel.core.bean.ResultBean;
 import com.java2nb.novel.core.utils.BeanUtil;
+import com.java2nb.novel.entity.CrawlSingleTask;
 import com.java2nb.novel.entity.CrawlSource;
 import com.java2nb.novel.service.CrawlService;
+import com.java2nb.novel.vo.CrawlSingleTaskVO;
 import com.java2nb.novel.vo.CrawlSourceVO;
 import lombok.RequiredArgsConstructor;
-import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -56,6 +57,38 @@ public class CrawlController {
         return ResultBean.ok();
     }
 
+    /**
+     * 新增单本采集任务
+     * */
+    @PostMapping("addCrawlSingleTask")
+    public ResultBean addCrawlSingleTask(CrawlSingleTask singleTask){
+        crawlService.addCrawlSingleTask(singleTask);
+
+        return ResultBean.ok();
+
+    }
+
+    /**
+     * 单本采集任务分页列表查询
+     * */
+    @PostMapping("listCrawlSingleTaskByPage")
+    public ResultBean listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
+
+        return ResultBean.ok(new PageInfo<>(BeanUtil.copyList(crawlService.listCrawlSingleTaskByPage(page,pageSize), CrawlSingleTaskVO.class)
+        ));
+    }
+
+    /**
+     * 删除采集任务
+     * */
+    @PostMapping("delCrawlSingleTask")
+    public ResultBean delCrawlSingleTask(Long id){
+
+        crawlService.delCrawlSingleTask(id);
+
+        return ResultBean.ok();
+    }
+
 
 
 

novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java

@@ -30,6 +30,8 @@ import static java.util.regex.Pattern.compile;
 @Slf4j
 public class CrawlParser {
 
+    private static IdWorker idWorker = new IdWorker();
+
     public static final Integer BOOK_INDEX_LIST_KEY = 1;
 
     public static final Integer BOOK_CONTENT_LIST_KEY = 2;
@@ -94,6 +96,12 @@ public class CrawlParser {
 
                     String desc = bookDetailHtml.substring(bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length());
                     desc = desc.substring(0, desc.indexOf(ruleBean.getDescEnd()));
+                    //过滤掉简介中的特殊标签
+                    desc = desc.replaceAll("<a[^<]+</a>","")
+                            .replaceAll("<font[^<]+</font>","")
+                            .replaceAll("<p>\\s*</p>","")
+                            .replaceAll("<p>","")
+                            .replaceAll("</p>","<br/>");
                     //设置书籍简介
                     book.setBookDesc(desc);
                     if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {
@@ -173,6 +181,7 @@ public class CrawlParser {
             String lastIndexName = null;
 
             while (isFindIndex) {
+
                 BookIndex hasIndex = hasIndexs.get(indexNum);
                 String indexName = indexNameMatch.group(1);
 
@@ -199,7 +208,7 @@ public class CrawlParser {
                         if(hasIndexs.size() == 0){
                             //新书入库
                             //设置目录和章节内容
-                            Long indexId = new IdWorker().nextId();
+                            Long indexId = idWorker.nextId();
                             lastIndexId = indexId;
                             lastIndexName = indexName;
                             bookIndex.setId(indexId);
@@ -232,8 +241,6 @@ public class CrawlParser {
                         }
 
 
-                    } else {
-                        break;
                     }
 
 

novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java

@@ -3,10 +3,7 @@ package com.java2nb.novel.core.listener;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.java2nb.novel.core.crawl.CrawlParser;
 import com.java2nb.novel.core.crawl.RuleBean;
-import com.java2nb.novel.entity.Book;
-import com.java2nb.novel.entity.BookContent;
-import com.java2nb.novel.entity.BookIndex;
-import com.java2nb.novel.entity.CrawlSource;
+import com.java2nb.novel.entity.*;
 import com.java2nb.novel.service.BookService;
 import com.java2nb.novel.service.CrawlService;
 import com.java2nb.novel.utils.Constants;
@@ -40,15 +37,15 @@ public class StarterListener implements ServletContextListener {
 
     @Override
     public void contextInitialized(ServletContextEvent sce) {
-        log.info("程序启动,开始执行自动更新线程。。。");
-        for(int i = 0 ; i<updateThreadCount; i++) {
+        for (int i = 0; i < updateThreadCount; i++) {
             new Thread(() -> {
+                log.info("程序启动,开始执行自动更新线程。。。");
                 while (true) {
                     try {
                         //1.查询最新目录更新时间在一个月之内的前100条需要更新的数据
                         Date currentDate = new Date();
                         Date startDate = DateUtils.addDays(currentDate, -30);
-                        List<Book> bookList ;
+                        List<Book> bookList;
                         synchronized (this) {
                             bookList = bookService.queryNeedUpdateBook(startDate, 100);
                         }
@@ -61,7 +58,7 @@ public class StarterListener implements ServletContextListener {
                                 Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
                                 //这里只做老书更新
                                 book.setId(needUpdateBook.getId());
-                                if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
+                                if (needUpdateBook.getPicUrl() != null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
                                     //本地图片则不更新
                                     book.setPicUrl(null);
                                 }
@@ -83,6 +80,42 @@ public class StarterListener implements ServletContextListener {
 
                 }
             }).start();
+
+
         }
+
+
+        new Thread(() -> {
+            log.info("程序启动,开始执行单本采集任务线程。。。");
+            while (true) {
+                CrawlSingleTask task = null;
+                byte crawlStatus = 0;
+                try {
+                    //获取采集任务
+                    task = crawlService.getCrawlSingleTask();
+
+                    if (task != null) {
+                        //查询爬虫规则
+                        CrawlSource source = crawlService.queryCrawlSource(task.getSourceId());
+                        RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
+
+                        if (crawlService.parseBookAndSave(task.getCatId(), ruleBean, task.getSourceId(), task.getSourceBookId())) {
+                            //采集成功
+                            crawlStatus = 1;
+                        }
+
+                    }
+
+                    Thread.sleep(1000 * 60);
+
+                } catch (Exception e) {
+                    log.error(e.getMessage(), e);
+                }
+                if (task != null) {
+                    crawlService.updateCrawlSingleTask(task, crawlStatus);
+                }
+
+            }
+        }).start();
     }
 }

novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java

@@ -1,6 +1,7 @@
 package com.java2nb.novel.service;
 
 import com.java2nb.novel.core.crawl.RuleBean;
+import com.java2nb.novel.entity.CrawlSingleTask;
 import com.java2nb.novel.entity.CrawlSource;
 
 import java.util.List;
@@ -39,6 +40,16 @@ public interface CrawlService {
      * */
     void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus);
 
+    /**
+     * 采集并保存小说
+     * @param catId 分类ID
+     * @param bookId 小说ID
+     * @param sourceId 源ID
+     * @param ruleBean 采集规则\
+     * @return true:成功，false:失败
+     * */
+    boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId);
+
     /**
      * 根据爬虫状态查询爬虫源集合
      * @param sourceStatus 状态，0关闭，1开启
@@ -61,4 +72,37 @@ public interface CrawlService {
      * @return 源信息
      * */
     CrawlSource queryCrawlSource(Integer sourceId);
+
+    /**
+     * 新增单本采集任务
+     * @param singleTask 任务信息对象
+     * */
+    void addCrawlSingleTask(CrawlSingleTask singleTask);
+
+    /**
+     * 单本采集任务分页列表查询
+     * @param page 当前页码
+     * @param pageSize 分页大小
+     * @return 单本采集任务集合
+     * */
+    List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize);
+
+    /**
+     * 删除采集任务
+     * @param id 任务ID
+     * */
+    void delCrawlSingleTask(Long id);
+
+    /**
+     * 获取采集任务
+     * @return 采集任务
+     * */
+    CrawlSingleTask getCrawlSingleTask();
+
+    /**
+     * 更新单本采集任务
+     * @param task 采集任务
+     * @param status 采集状态
+     * */
+    void updateCrawlSingleTask(CrawlSingleTask task, Byte status);
 }

novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java

@@ -6,12 +6,12 @@ import com.java2nb.novel.core.cache.CacheKey;
 import com.java2nb.novel.core.cache.CacheService;
 import com.java2nb.novel.core.crawl.CrawlParser;
 import com.java2nb.novel.core.crawl.RuleBean;
+import com.java2nb.novel.core.enums.ResponseStatus;
+import com.java2nb.novel.core.exception.BusinessException;
 import com.java2nb.novel.core.utils.IdWorker;
 import com.java2nb.novel.core.utils.SpringUtil;
 import com.java2nb.novel.core.utils.ThreadUtil;
-import com.java2nb.novel.entity.Book;
-import com.java2nb.novel.entity.BookContent;
-import com.java2nb.novel.entity.BookIndex;
+import com.java2nb.novel.entity.*;
 import com.java2nb.novel.entity.CrawlSource;
 import com.java2nb.novel.mapper.*;
 import com.java2nb.novel.service.BookService;
@@ -33,8 +33,7 @@ import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClient;
 import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlBookId;
 import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlSourceId;
 import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*;
-import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
-import static org.mybatis.dynamic.sql.SqlBuilder.update;
+import static org.mybatis.dynamic.sql.SqlBuilder.*;
 import static org.mybatis.dynamic.sql.select.SelectDSL.select;
 
 /**
@@ -48,6 +47,8 @@ public class CrawlServiceImpl implements CrawlService {
 
     private final CrawlSourceMapper crawlSourceMapper;
 
+    private final CrawlSingleTaskMapper crawlSingleTaskMapper;
+
     private final BookService bookService;
 
 
@@ -140,6 +141,62 @@ public class CrawlServiceImpl implements CrawlService {
         return crawlSourceMapper.selectMany(render).get(0);
     }
 
+    @Override
+    public void addCrawlSingleTask(CrawlSingleTask singleTask) {
+
+        if(bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(),singleTask.getAuthorName())){
+            throw new BusinessException(ResponseStatus.BOOK_EXISTS);
+
+        }
+        singleTask.setCreateTime(new Date());
+        crawlSingleTaskMapper.insertSelective(singleTask);
+
+
+    }
+
+    @Override
+    public List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize) {
+        PageHelper.startPage(page, pageSize);
+        SelectStatementProvider render = select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
+                .from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
+                .orderBy(CrawlSingleTaskDynamicSqlSupport.createTime.descending())
+                .build()
+                .render(RenderingStrategies.MYBATIS3);
+        return crawlSingleTaskMapper.selectMany(render);
+    }
+
+    @Override
+    public void delCrawlSingleTask(Long id) {
+        crawlSingleTaskMapper.deleteByPrimaryKey(id);
+    }
+
+    @Override
+    public CrawlSingleTask getCrawlSingleTask() {
+
+         List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
+                .from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
+                .where(CrawlSingleTaskDynamicSqlSupport.taskStatus,isEqualTo((byte)2))
+                 .orderBy(CrawlSingleTaskDynamicSqlSupport.createTime)
+                 .limit(1)
+                .build()
+                .render(RenderingStrategies.MYBATIS3));
+
+         return list.size() > 0 ? list.get(0) : null;
+    }
+
+    @Override
+    public void updateCrawlSingleTask(CrawlSingleTask task, Byte status) {
+        byte excCount = task.getExcCount();
+        excCount+=1;
+        task.setExcCount(excCount);
+        if(status == 1 || excCount == 5){
+            //当采集成功或者采集次数等于5，则更新采集最终状态，并停止采集
+            task.setTaskStatus(status);
+        }
+        crawlSingleTaskMapper.updateByPrimaryKeySelective(task);
+
+    }
+
     /**
      * 解析分类列表
      */
@@ -167,36 +224,13 @@ public class CrawlServiceImpl implements CrawlService {
                         boolean isFindBookId = bookIdMatcher.find();
                         while (isFindBookId) {
                             try {
-                                String bookId = bookIdMatcher.group(1);
-                                Book book = CrawlParser.parseBook(ruleBean, bookId);
-                                //这里只做新书入库，查询是否存在这本书
-                                Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
-                                //如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库
-                                if (existBook == null) {
-                                    //没有该书，可以入库
-                                    book.setCatId(catId);
-                                    //根据分类ID查询分类
-                                    book.setCatName(bookService.queryCatNameByCatId(catId));
-                                    if (catId == 7) {
-                                        //女频
-                                        book.setWorkDirection((byte) 1);
-                                    } else {
-                                        //男频
-                                        book.setWorkDirection((byte) 0);
-                                    }
-                                    book.setCrawlBookId(bookId);
-                                    book.setCrawlSourceId(sourceId);
-                                    book.setCrawlLastTime(new Date());
-                                    book.setId(new IdWorker().nextId());
-                                    //解析章节目录
-                                    Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
-
-                                    bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
-
-                                } else {
-                                    //只更新书籍的爬虫相关字段
-                                    bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
+                                if(Thread.currentThread().isInterrupted()){
+                                    return;
                                 }
+
+
+                                String bookId = bookIdMatcher.group(1);
+                                parseBookAndSave(catId, ruleBean, sourceId, bookId);
                             } catch (Exception e) {
                                 log.error(e.getMessage(), e);
                             }
@@ -227,6 +261,43 @@ public class CrawlServiceImpl implements CrawlService {
 
     }
 
+    @Override
+    public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) {
+        Book book = CrawlParser.parseBook(ruleBean, bookId);
+        if(book.getBookName() == null || book.getAuthorName() == null){
+            return false;
+        }
+        //这里只做新书入库，查询是否存在这本书
+        Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
+        //如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库
+        if (existBook == null) {
+            //没有该书，可以入库
+            book.setCatId(catId);
+            //根据分类ID查询分类
+            book.setCatName(bookService.queryCatNameByCatId(catId));
+            if (catId == 7) {
+                //女频
+                book.setWorkDirection((byte) 1);
+            } else {
+                //男频
+                book.setWorkDirection((byte) 0);
+            }
+            book.setCrawlBookId(bookId);
+            book.setCrawlSourceId(sourceId);
+            book.setCrawlLastTime(new Date());
+            book.setId(new IdWorker().nextId());
+            //解析章节目录
+            Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
+
+            bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
+
+        } else {
+            //只更新书籍的爬虫相关字段
+            bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
+        }
+        return true;
+    }
+
     @Override
     public void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus) {
         CrawlSource source = new CrawlSource();

novel-crawl/src/main/java/com/java2nb/novel/vo/CrawlSingleTaskVO.java

@@ -0,0 +1,26 @@
+package com.java2nb.novel.vo;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.java2nb.novel.entity.CrawlSingleTask;
+import com.java2nb.novel.entity.CrawlSource;
+import lombok.Data;
+
+import java.util.Date;
+
+/**
+ * @author Administrator
+ */
+@Data
+public class CrawlSingleTaskVO extends CrawlSingleTask {
+
+    @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm")
+    private Date createTime;
+
+
+
+
+    @Override
+    public String toString() {
+        return super.toString();
+    }
+}

novel-crawl/src/main/resources/static/css/user.css

@@ -10,7 +10,7 @@
 .user_l .log_list { width:350px }
 .user_l .s_input { margin-bottom:25px; font-size:14px }
 .s_input { width:348px; height:30px; line-height:38px\9; vertical-align:middle; border:1px solid #ddd; border-radius:2px }
-.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px; background:url(../images/icon_user.png) no-repeat 13px 13px }
+.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px}
 .icon_key { background-position: 13px -51px }
 .icon_code { background-position: 13px -117px; width:200px; float:left }
 .code_pic { height:38px; float:right }
@@ -37,7 +37,7 @@
 .fast_tit .title { background:#fff; font-size:16px; padding:3px 14px; position:relative; display:inline-block; z-index:999 }
 /*userinfo*/
 .my_l { width:198px; float:left; font-size: 13px; padding-top: 20px; }
-.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff; background:url(../images/icon_user.png) no-repeat; margin-bottom:5px; color: #666 }
+.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff;  margin-bottom:5px; color: #666 }
 .my_l li .on { background-color:#fafafa; border-left:2px solid #f80; color:#000; border-radius: 0 2px 2px 0 }
 .my_l .link_1 { background-position:32px -188px }
 .my_l .link_2 { background-position:32px -230px }

novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_add.html

@@ -0,0 +1,188 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
+        <title>爬虫管理系统-小说精品屋</title>
+        <link rel="stylesheet" href="/css/base.css?v=1"/>
+        <link rel="stylesheet" href="/css/user.css"/>
+    </head>
+</head>
+<body class="">
+
+<div class="header">
+    <div class="mainNav" id="mainNav">
+        <div class="box_center cf"
+             style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
+
+            小说精品屋爬虫管理
+
+        </div>
+    </div>
+</div>
+
+<div class="main box_center cf">
+    <div class="userBox cf">
+        <div class="my_l">
+
+            <ul class="log_list">
+                <li><a class="link_1" href="/">爬虫源管理</a></li>
+                <li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
+                <!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
+                <li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
+            </ul>
+
+        </div>
+        <div class="my_r">
+            <div class="my_bookshelf">
+
+                <div class="userBox cf">
+                    <form method="post" action="./register.html" id="form2">
+
+                        <div class="user_l">
+                            <div></div>
+                            <h3>采集信息填写（示例均为笔趣阁：http://www.mcmssc.com）</h3>
+                            <ul class="log_list">
+                                <li><span id="LabErr"></span></li>
+                                <input type="hidden" id="sourceName" class="s_input icon_name" >
+                                <b>采集源:</b>
+                                <li><select id="sourceId" class="s_input icon_key">
+                                </select></li>
+                                <b>采集分类:</b>
+                                <li><select id="catId" class="s_input icon_key">
+                                    <option value="1">玄幻奇幻</option>
+                                    <option value="2">武侠仙侠</option>
+                                    <option value="3">都市言情</option>
+                                    <option value="4">历史军事</option>
+                                    <option value="5">科幻灵异</option>
+                                    <option value="6">网游竞技</option>
+                                    <option value="7">女生频道</option>
+                                </select>
+                                </li>
+                                示例：<b>73_73911</b>
+                                <li><input type="text" id="sourceBookId" class="s_input icon_key"
+                                           placeholder="采集的源站小说ID"/></li>
+                                示例：<b>苏厨</b>
+                                <li><input type="text" id="bookName" class="s_input icon_key"
+                                           placeholder="采集的小说名"/></li>
+                                示例：<b>二子从周</b>
+                                <li><input type="text" id="authorName" class="s_input icon_key"
+                                           placeholder="采集的小说作者名"/></li>
+
+                                <!--示例：<b>https://m.xdingdiann.com/sort/0/1.html</b>
+                                <li><input type="text" id="updateBookListUrl" class="s_input icon_key"
+                                           placeholder="小说更新列表url"></li>-->
+
+
+                                <li><input type="button" onclick="addCrawlSingleTask()" name="btnRegister" value="提交"
+                                           id="btnRegister" class="btn_red"></li>
+                            </ul>
+                        </div>
+                    </form>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+
+</body>
+<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
+<script src="/layui/layui.all.js" type="text/javascript"></script>
+<script src="/javascript/header.js" type="text/javascript"></script>
+<script src="/javascript/user.js" type="text/javascript"></script>
+<script language="javascript" type="text/javascript">
+    $(function () {
+        $.ajax({
+            type: "POST",
+            url: "/crawl/listCrawlByPage",
+            data: {'curr':1,'limit':100},
+            dataType: "json",
+            success: function (data) {
+                if (data.code == 200) {
+                    var crawlSourceList = data.data.list;
+                    if (crawlSourceList.length > 0) {
+                        var crawlSourceListHtml = "";
+                        for(var i=0;i<crawlSourceList.length;i++){
+                            var crawlSource = crawlSourceList[i];
+                            crawlSourceListHtml+=("<option value=\""+crawlSource.id+"\">"+crawlSource.sourceName+"</option>");
+                        }
+                        $("#sourceId").html(crawlSourceListHtml);
+
+
+                    }
+
+
+                } else if (data.code == 1001) {
+                    //未登录
+                    location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href);
+
+                }else {
+                    layer.alert(data.msg);
+                }
+
+            },
+            error: function () {
+                layer.alert('网络异常');
+            }
+        })
+    });
+
+
+    function addCrawlSingleTask() {
+
+        var sourceId = $("#sourceId").find("option:selected").val();
+
+        var catId = $("#catId").find("option:selected").val();
+
+        var sourceName = $("#sourceId").find("option:selected").html();
+
+
+        var sourceBookId = $("#sourceBookId").val();
+
+        if (!sourceBookId){
+            layer.alert("采集的源站小说ID不能为空");
+            return;
+        }
+
+        var bookName = $("#bookName").val();
+
+        if (!bookName){
+            layer.alert("采集的小说名不能为空");
+            return;
+        }
+
+        var authorName = $("#authorName").val();
+
+        if (!authorName){
+            layer.alert("采集的小说作者名不能为空");
+            return;
+        }
+
+        $.ajax({
+            type: "POST",
+            url: "/crawl/addCrawlSingleTask",
+            data: {'sourceId': sourceId, 'sourceName': sourceName,'catId':catId,'sourceBookId' : sourceBookId,'bookName':bookName,'authorName':authorName},
+            dataType: "json",
+            success: function (data) {
+                if (data.code == 200) {
+
+                    window.location.href = '/crawl/crawlSingleTask_list.html';
+
+
+                } else {
+                    layer.alert(data.msg);
+                }
+
+            },
+            error: function () {
+                layer.alert('网络异常');
+            }
+        })
+
+    }
+
+</script>
+</html>

novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html

@@ -0,0 +1,230 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
+        <title>爬虫管理系统-小说精品屋</title>
+        <link rel="stylesheet" href="/css/base.css?v=1"/>
+        <link rel="stylesheet" href="/css/user.css"/>
+    </head>
+</head>
+<body class="">
+
+<div class="header">
+    <div class="mainNav" id="mainNav">
+        <div class="box_center cf"
+             style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
+
+            小说精品屋爬虫管理
+
+        </div>
+    </div>
+</div>
+
+<div class="main box_center cf">
+    <div class="userBox cf">
+        <div class="my_l">
+
+            <ul class="log_list">
+                <li><a class="link_1" href="/">爬虫源管理</a></li>
+                <li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
+                <!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
+                 <li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
+            </ul>
+
+        </div>
+        <div class="my_r">
+            <div class="my_bookshelf">
+                <div class="title cf">
+                    <h2 class="fl">单本采集任务列表</h2>
+                    <div class="fr"><a href="/crawl/crawlSingleTask_add.html" class="btn_red">增加单本采集任务</a></div>
+                </div>
+
+                <div id="divData" class="updateTable">
+                    <table cellpadding="0" cellspacing="0">
+                        <thead>
+                        <tr>
+                            <th class="style">
+                                序号
+                            </th>
+                            <th class="name">
+                                采集小说名
+                            </th>
+                            <th class="name">
+                                采集小说作者名
+                            </th>
+                            <th class="goread">
+                                采集次数
+                            </th>
+                            <th class="goread">
+                                状态
+                            </th>
+                            <th class="name">
+                                创建时间
+                            </th>
+                            <th class="goread">
+                                操作
+                            </th>
+                        </tr>
+                        </thead>
+                        <tbody id="crawlSourceList">
+
+
+                        </tbody>
+                    </table>
+                    <div class="pageBox cf" id="shellPage">
+                    </div>
+                </div>
+                <!--<div id="divData" class="updateTable">
+                    <table cellpadding="0" cellspacing="0">
+                        <thead>
+                        <tr>
+
+                            <th class="name">
+                                爬虫源（已开启的爬虫源）
+                            </th>
+                            <th class="chapter">
+                                成功爬取数量（websocket实现）
+                            </th>
+                            <th class="time">
+                            目标爬取数量
+                            </th>
+                            <th class="goread">
+                                状态（正在运行，已停止）（一次只能运行一个爬虫源）
+                            </th>
+                            <th class="goread">
+                                操作（启动，停止）
+                            </th>
+                        </tr>
+                        </thead>
+                        <tbody id="bookShelfList">
+
+
+
+                        </tbody>
+                    </table>
+                    <div class="pageBox cf" id="shellPage">
+                    </div>
+                </div>-->
+            </div>
+        </div>
+    </div>
+</div>
+
+
+</body>
+<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
+<script src="/layui/layui.all.js" type="text/javascript"></script>
+<script src="/javascript/header.js" type="text/javascript"></script>
+<script src="/javascript/user.js" type="text/javascript"></script>
+<script language="javascript" type="text/javascript">
+    search(1, 10);
+
+    function search(curr, limit) {
+
+        $.ajax({
+            type: "POST",
+            url: "/crawl/listCrawlSingleTaskByPage",
+            data: {'curr': curr, 'limit': limit},
+            dataType: "json",
+            success: function (data) {
+                if (data.code == 200) {
+                    var crawlSourceList = data.data.list;
+                    if (crawlSourceList.length > 0) {
+                        var crawlSourceListHtml = "";
+                        for (var i = 0; i < crawlSourceList.length; i++) {
+                            var crawlSource = crawlSourceList[i];
+                            crawlSourceListHtml += (" <tr class=\"book_list\" vals=\"291\">\n" +
+                                "                            <td class=\"style bookclass\">\n" +
+                                "                                [" + (i + 1) + "]\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"name\">\n" +
+                                "                                " + crawlSource.bookName + "\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"name\">\n" +
+                                "                                " + crawlSource.authorName + "\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"goread\">\n" +
+                                "                                " + crawlSource.excCount + "\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"goread\">\n" +
+                                "                                " + (crawlSource.taskStatus == 0 ? '采集失败' : (crawlSource.taskStatus == 1 ? '采集成功' : (crawlSource.excCount>0?'采集中':'排队中'))) + "\n" +
+                                "                            </td>\n" +
+                                "                            <td class=\"name\" valsc=\"291|2037554|1\">"
+                                + crawlSource.createTime + "</td>\n" +
+
+                                " <td class=\"goread\" id='opt" + crawlSource.id + "'><a href='javascript:del(" + crawlSource.id + ")'>删除  </a></td>                     </tr>");
+                        }
+                        $("#crawlSourceList").html(crawlSourceListHtml);
+
+                        layui.use('laypage', function () {
+                            var laypage = layui.laypage;
+
+                            //执行一个laypage实例
+                            laypage.render({
+                                elem: 'shellPage' //注意，这里的 test1 是 ID，不用加 # 号
+                                , count: data.data.total //数据总数，从服务端得到,
+                                , curr: data.data.pageNum
+                                , limit: data.data.pageSize
+                                , jump: function (obj, first) {
+
+
+                                    //obj包含了当前分页的所有参数，比如：
+                                    console.log(obj.curr); //得到当前页，以便向服务端请求对应页的数据。
+                                    console.log(obj.limit); //得到每页显示的条数
+
+
+                                    //首次不执行
+                                    if (!first) {
+                                        search(obj.curr, obj.limit);
+                                    } else {
+
+                                    }
+                                }
+                            });
+                        });
+
+                    }
+
+
+                }  else {
+                    layer.alert(data.msg);
+                }
+
+            },
+            error: function () {
+                layer.alert('网络异常');
+            }
+        })
+
+    }
+
+
+    function del(id) {
+
+        $.ajax({
+            type: "POST",
+            url: "/crawl/delCrawlSingleTask",
+            data: {'id': id},
+            dataType: "json",
+            success: function (data) {
+                if (data.code == 200) {
+                    location.reload();
+
+
+                } else {
+                    layer.alert(data.msg);
+                }
+
+            },
+            error: function () {
+                layer.alert('网络异常');
+            }
+        })
+
+
+    }
+</script>
+</html>

novel-crawl/src/main/resources/templates/crawl/crawlSource_add.html

@@ -29,6 +29,7 @@
 
             <ul class="log_list">
                 <li><a class="link_1 on" href="/">爬虫源管理</a></li>
+                <li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
                 <!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
                 <li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
             </ul>

novel-crawl/src/main/resources/templates/crawl/crawlSource_list.html

@@ -28,6 +28,7 @@
 
             <ul class="log_list">
                 <li><a class="link_1 on" href="/">爬虫源管理</a></li>
+                <li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
                <!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
                 <li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
             </ul>

novel-front/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.0.2</version>
+        <version>2.6.4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -27,11 +27,57 @@
             <version>${jjwt.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-amqp</artifactId>
+        </dependency>
+
+
+        <dependency>
+            <groupId>io.searchbox</groupId>
+            <artifactId>jest</artifactId>
+            <version>${jest.version}</version>
+        </dependency>
+
+        <!--aliyunOSS-->
+        <dependency>
+            <groupId>com.aliyun.oss</groupId>
+            <artifactId>aliyun-sdk-oss</artifactId>
+            <version>${aliyun-sdk-oss.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-fileupload</groupId>
+            <artifactId>commons-fileupload</artifactId>
+            <version>${commons-fileupload.version}</version>
+        </dependency>
+
+        <!--FastDfs-->
+        <dependency>
+            <groupId>com.github.tobato</groupId>
+            <artifactId>fastdfs-client</artifactId>
+            <version>${fastdfs-client.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
+        </dependency>
+
+
+
+
+        <!--引入redisson分布式锁-->
+       <!-- <dependency>
+            <groupId>org.redisson</groupId>
+            <artifactId>redisson-spring-boot-starter</artifactId>
+            <version>${redisson.version}</version>
+        </dependency>-->
+
 
         <dependency>
             <groupId>com.alipay.sdk</groupId>
             <artifactId>alipay-sdk-java</artifactId>
-            <version>4.9.153.ALL</version>
+            <version>${alipay-sdk-java.version}</version>
         </dependency>
     </dependencies>
 

novel-front/src/main/java/com/java2nb/novel/FrontNovelApplication.java

@@ -1,10 +1,13 @@
 package com.java2nb.novel;
 
+import com.github.tobato.fastdfs.FdfsClientConfig;
+import io.shardingsphere.shardingjdbc.spring.boot.SpringBootConfiguration;
 import org.mybatis.spring.annotation.MapperScan;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.TaskScheduler;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
@@ -18,6 +21,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
 @EnableScheduling
 @EnableCaching
 @MapperScan(basePackages = {"com.java2nb.novel.mapper"})
+@Import(FdfsClientConfig.class)
 public class FrontNovelApplication {
 
     public static void main(String[] args) {

novel-front/src/main/java/com/java2nb/novel/controller/AuthorController.java

@@ -55,7 +55,7 @@ public class AuthorController extends BaseController{
      * 发布小说
      * */
     @PostMapping("addBook")
-    public ResultBean addBook(Book book,HttpServletRequest request){
+    public ResultBean addBook(@RequestParam("bookDesc") String bookDesc,Book book,HttpServletRequest request){
 
         //查询作家信息
         Author author = authorService.queryAuthor(getUserDetails(request).getId());
@@ -67,6 +67,10 @@ public class AuthorController extends BaseController{
 
         }
 
+        //bookDesc不能使用book对象来接收，否则会自动去掉前面的空格
+        book.setBookDesc(bookDesc
+                .replaceAll("\\n","<br>")
+                .replaceAll("\\s","&nbsp;"));
         //发布小说
         bookService.addBook(book,author.getId(),author.getPenName());
 
@@ -109,6 +113,8 @@ public class AuthorController extends BaseController{
             return ResultBean.fail(ResponseStatus.AUTHOR_STATUS_FORBIDDEN);
         }
 
+        content = content.replaceAll("\\n","<br>")
+                .replaceAll("\\s","&nbsp;");
         //发布章节内容
         bookService.addBookContent(bookId,indexName,content,author.getId());
 

novel-front/src/main/java/com/java2nb/novel/controller/BookController.java

@@ -4,14 +4,14 @@ import com.github.pagehelper.PageInfo;
 import com.java2nb.novel.core.bean.ResultBean;
 import com.java2nb.novel.core.bean.UserDetails;
 import com.java2nb.novel.core.enums.ResponseStatus;
-import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.entity.BookComment;
-import com.java2nb.novel.entity.BookIndex;
 import com.java2nb.novel.search.BookSP;
 import com.java2nb.novel.service.BookService;
 import com.java2nb.novel.vo.BookVO;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -19,7 +19,6 @@ import org.springframework.web.bind.annotation.RestController;
 
 import javax.servlet.http.HttpServletRequest;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -33,6 +32,11 @@ public class BookController extends BaseController{
 
     private final BookService bookService;
 
+    private final RabbitTemplate rabbitTemplate;
+
+    @Value("${spring.rabbitmq.enable}")
+    private Integer enableMq;
+
 
     /**
      * 查询首页小说设置列表数据
@@ -79,8 +83,8 @@ public class BookController extends BaseController{
      * */
     @PostMapping("searchByPage")
     public ResultBean searchByPage(BookSP bookSP, @RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize){
-        List<BookVO> books = bookService.searchByPage(bookSP,page,pageSize);
-        return ResultBean.ok(new PageInfo<>(books));
+        PageInfo<BookVO> pageInfo = bookService.searchByPage(bookSP,page,pageSize);
+        return ResultBean.ok(pageInfo);
     }
 
     /**
@@ -105,7 +109,11 @@ public class BookController extends BaseController{
      * */
     @PostMapping("addVisitCount")
     public ResultBean addVisitCount(Long bookId){
-        bookService.addVisitCount(bookId);
+        if(enableMq == 1) {
+            rabbitTemplate.convertAndSend("ADD-BOOK-VISIT-EXCHANGE", null, bookId);
+        }else {
+            bookService.addVisitCount(bookId, 1);
+        }
         return ResultBean.ok();
     }
 

novel-front/src/main/java/com/java2nb/novel/controller/FileController.java

@@ -1,17 +1,32 @@
 package com.java2nb.novel.controller;
 
+import com.java2nb.novel.core.bean.ResultBean;
 import com.java2nb.novel.core.cache.CacheService;
+import com.java2nb.novel.core.utils.Constants;
 import com.java2nb.novel.core.utils.RandomValidateCodeUtil;
+import com.java2nb.novel.core.utils.RestTemplateUtil;
+import com.java2nb.novel.core.utils.UUIDUtil;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.Charsets;
+import org.apache.http.client.utils.DateUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.Resource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
 
+import javax.imageio.ImageIO;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
+import java.io.*;
+import java.util.Date;
+import java.util.Objects;
 
 /**
  * @author 11797
@@ -24,6 +39,9 @@ public class FileController {
 
     private final CacheService cacheService;
 
+    @Value("${pic.save.path}")
+    private String picSavePath;
+
     /**
      * 生成验证码
      */
@@ -44,5 +62,32 @@ public class FileController {
         }
     }
 
+    /**
+     * 文件上传
+     */
+    @ResponseBody
+    @PostMapping("/upload")
+    ResultBean upload(@RequestParam("file") MultipartFile file) {
+        Date currentDate = new Date();
+        try {
+            String savePath =
+                    Constants.LOCAL_PIC_PREFIX + DateUtils.formatDate(currentDate, "yyyy") + "/" +
+                    DateUtils.formatDate(currentDate, "MM") + "/" +
+                    DateUtils.formatDate(currentDate, "dd") ;
+            String oriName = file.getOriginalFilename();
+            String saveFileName = UUIDUtil.getUUID32() + oriName.substring(oriName.lastIndexOf("."));
+            File saveFile = new File( picSavePath + savePath, saveFileName);
+            if (!saveFile.getParentFile().exists()) {
+                saveFile.getParentFile().mkdirs();
+            }
+            file.transferTo(saveFile);
+            return ResultBean.ok(savePath+"/"+saveFileName);
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+            return ResultBean.error();
+        }
+
+    }
+
 
 }

novel-front/src/main/java/com/java2nb/novel/core/config/FdfsConfiguration.java

@@ -0,0 +1,13 @@
+package com.java2nb.novel.core.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.EnableMBeanExport;
+import org.springframework.jmx.support.RegistrationPolicy;
+
+/**
+ * @author 11797
+ */
+@Configuration
+@EnableMBeanExport(registration= RegistrationPolicy.IGNORE_EXISTING)
+public class FdfsConfiguration {
+}

novel-front/src/main/java/com/java2nb/novel/core/config/OssProperties.java

@@ -0,0 +1,30 @@
+package com.java2nb.novel.core.config;
+
+import lombok.Data;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author 11797
+ */
+@Data
+@Component
+@ConfigurationProperties(prefix="novel.file")
+public class OssProperties{
+
+    private String endpoint;
+
+    private String keyId;
+
+    private String keySecret;
+
+    private String fileHost;
+
+    private String bucketName;
+
+    private String webUrl;
+
+
+}

novel-front/src/main/java/com/java2nb/novel/core/config/RabbitConfig.java

@@ -0,0 +1,59 @@
+package com.java2nb.novel.core.config;
+
+import org.springframework.amqp.core.*;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author 11797
+ */
+@Configuration
+@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "enable", havingValue = "1")
+public class RabbitConfig {
+
+
+    /**
+     * 更新数据库队列
+     */
+    @Bean
+    public Queue updateDbQueue() {
+        return new Queue("UPDATE-DB-QUEUE", true);
+    }
+
+    /**
+     * 更新数据库队列
+     */
+    @Bean
+    public Queue updateEsQueue() {
+        return new Queue("UPDATE-ES-QUEUE", true);
+    }
+
+
+    /**
+     * 增加点击量交换机
+     */
+    @Bean
+    public FanoutExchange addVisitExchange() {
+        return new FanoutExchange("ADD-BOOK-VISIT-EXCHANGE");
+    }
+
+    /**
+     * 更新搜索引擎队列绑定到增加点击量交换机中
+     */
+    @Bean
+    public Binding updateEsBinding() {
+
+        return BindingBuilder.bind(updateEsQueue()).to(addVisitExchange());
+    }
+
+    /**
+     * 更新数据库绑定到增加点击量交换机中
+     */
+    @Bean
+    public Binding updateDbBinding() {
+        return BindingBuilder.bind(updateDbQueue()).to(addVisitExchange());
+    }
+
+
+}

novel-front/src/main/java/com/java2nb/novel/core/listener/BookVisitAddListener.java

@@ -0,0 +1,107 @@
+package com.java2nb.novel.core.listener;
+
+import com.java2nb.novel.core.cache.CacheKey;
+import com.java2nb.novel.core.cache.CacheService;
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.service.BookService;
+import com.java2nb.novel.service.SearchService;
+import com.java2nb.novel.vo.EsBookVO;
+import com.rabbitmq.client.Channel;
+import io.searchbox.client.JestClient;
+import io.searchbox.core.Index;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.core.Message;
+import org.springframework.beans.BeanUtils;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+
+
+import java.text.SimpleDateFormat;
+
+
+/**
+ * @author 11797
+ */
+@Component
+@Slf4j
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "enable", havingValue = "1")
+public class BookVisitAddListener {
+
+    private final BookService bookService;
+
+    private final CacheService cacheService;
+
+    private final SearchService searchService;
+
+   // private final RedissonClient redissonClient;
+
+
+
+
+    /**
+     * 更新数据库
+     * 流量削峰，每本小说累积10个点击更新一次
+     */
+    @SneakyThrows
+    @RabbitListener(queues = {"UPDATE-DB-QUEUE"})
+    public void updateDb(Long bookId, Channel channel, Message message) {
+
+        log.debug("收到更新数据库消息：" + bookId);
+        Thread.sleep(1000 * 2);
+        //TODO 操作共享资源visitCount，集群环境下有线程安全问题，引入Redisson框架实现分布式锁
+        //RLock lock = redissonClient.getLock("visitCount");
+        //lock.lock();
+
+
+        //目前visitCount不重要，数据可丢失，暂不实现分布式锁
+        Integer visitCount = (Integer) cacheService.getObject(CacheKey.BOOK_ADD_VISIT_COUNT+bookId);
+        if(visitCount == null){
+            visitCount = 0 ;
+        }
+        cacheService.setObject(CacheKey.BOOK_ADD_VISIT_COUNT+bookId,++visitCount);
+        if(visitCount >= Constants.ADD_MAX_VISIT_COUNT) {
+            bookService.addVisitCount(bookId,visitCount);
+            cacheService.del(CacheKey.BOOK_ADD_VISIT_COUNT+bookId);
+        }
+
+        //TODO 操作共享资源visitCount，集群环境下有线程安全问题，引入Redisson框架实现分布式锁
+        //lock.unlock();
+
+
+
+    }
+
+    /**
+     * 更新搜索引擎
+     * 流量削峰，每本小说1个小时更新一次
+     */
+    @RabbitListener(queues = {"UPDATE-ES-QUEUE"})
+    public void updateEs(Long bookId, Channel channel, Message message) {
+
+        log.debug("收到更新搜索引擎消息：" + bookId);
+        if (cacheService.get(CacheKey.ES_IS_UPDATE_VISIT + bookId) == null) {
+            cacheService.set(CacheKey.ES_IS_UPDATE_VISIT + bookId, "1", 60 * 60);
+            try {
+                Thread.sleep(1000 * 5);
+                Book book = bookService.queryBookDetail(bookId);
+                searchService.importToEs(book);
+            }catch (Exception e){
+                cacheService.del(CacheKey.ES_IS_UPDATE_VISIT + bookId);
+                log.error("更新搜索引擎失败"+bookId);
+            }
+
+
+
+
+        }
+
+
+    }
+
+
+}

novel-front/src/main/java/com/java2nb/novel/core/schedule/BookToEsSchedule.java

@@ -0,0 +1,87 @@
+package com.java2nb.novel.core.schedule;
+
+import com.java2nb.novel.core.cache.CacheKey;
+import com.java2nb.novel.core.cache.CacheService;
+import com.java2nb.novel.core.utils.BeanUtil;
+import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.service.BookService;
+import com.java2nb.novel.service.SearchService;
+import com.java2nb.novel.vo.EsBookVO;
+import io.searchbox.client.JestClient;
+import io.searchbox.core.DocumentResult;
+import io.searchbox.core.Index;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * 小说导入搜索引擎
+ *
+ * @author Administrator
+ */
+@ConditionalOnProperty(prefix = "spring.elasticsearch", name = "enable", havingValue = "1")
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class BookToEsSchedule {
+
+    private final BookService bookService;
+
+    private final CacheService cacheService;
+
+
+
+    private final SearchService searchService;
+
+
+    /**
+     * 1分钟导入一次
+     */
+    @Scheduled(fixedRate = 1000 * 60)
+    public void saveToEs() {
+        //TODO 引入Redisson框架实现分布式锁
+        //可以重复更新，只是效率可能略有降低，所以暂不实现分布式锁
+        if (cacheService.get(CacheKey.ES_TRANS_LOCK) == null) {
+            cacheService.set(CacheKey.ES_TRANS_LOCK, "1", 60 * 20);
+            try {
+                //查询需要更新的小说
+                Date lastDate = (Date) cacheService.getObject(CacheKey.ES_LAST_UPDATE_TIME);
+                if (lastDate == null) {
+                    lastDate = new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-01");
+                }
+
+
+                List<Book> books = bookService.queryBookByUpdateTimeByPage(lastDate, 100);
+                for (Book book : books) {
+                    searchService.importToEs(book);
+                    lastDate = book.getUpdateTime();
+                    Thread.sleep(5000);
+
+                }
+
+
+
+                cacheService.setObject(CacheKey.ES_LAST_UPDATE_TIME, lastDate);
+
+            } catch (Exception e) {
+                log.error(e.getMessage(), e);
+            }
+            cacheService.del(CacheKey.ES_TRANS_LOCK);
+
+
+        }
+
+
+    }
+
+
+}

novel-front/src/main/java/com/java2nb/novel/core/schedule/CrawlPicTransSchedule.java

@@ -1,8 +1,10 @@
 package com.java2nb.novel.core.schedule;
 
+import com.java2nb.novel.core.utils.Constants;
 import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.service.BookService;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -12,14 +14,15 @@ import org.springframework.stereotype.Service;
 import java.util.List;
 
 /**
- * 网络图片转存本地任务
+ * 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）任务
+ *
  * @author Administrator
  */
-@ConditionalOnProperty(prefix = "pic.save",name = "type",havingValue = "2")
+@ConditionalOnProperty(prefix = "pic.save", name = "type", havingValue = "2")
 @Service
 @RequiredArgsConstructor
 @Slf4j
-public class Network2LocalPicSchedule {
+public class CrawlPicTransSchedule {
 
     private final BookService bookService;
 
@@ -33,20 +36,18 @@ public class Network2LocalPicSchedule {
      * 10分钟转一次
      */
     @Scheduled(fixedRate = 1000 * 60 * 10)
+    @SneakyThrows
     public void trans() {
 
         log.info("Network2LocalPicSchedule。。。。。。。。。。。。");
 
 
-        Integer offset = 0, limit = 100;
-        List<Book> networkPicBooks;
-        do {
-            networkPicBooks = bookService.queryNetworkPicBooks(limit, offset);
-            for (Book book : networkPicBooks) {
-                bookService.updateBookPicToLocal(book.getPicUrl(), book.getId());
-            }
-            offset += limit;
-        } while (networkPicBooks.size() > 0);
+        List<Book> networkPicBooks = bookService.queryNetworkPicBooks(Constants.LOCAL_PIC_PREFIX,100);
+        for (Book book : networkPicBooks) {
+            bookService.updateBookPicToLocal(book.getPicUrl(), book.getId());
+            //3秒钟转化一张图片，10分钟转化200张
+            Thread.sleep(3000);
+        }
 
 
     }

novel-front/src/main/java/com/java2nb/novel/core/utils/BrowserUtil.java

@@ -18,19 +18,20 @@ public class BrowserUtil {
             "lg-d", "lg-g", "lge-", "maui", "maxo", "midp", "mits", "mmef", "mobi", "mot-", "moto", "mwbp", "nec-",
             "newt", "noki", "oper", "palm", "pana", "pant", "phil", "play", "port", "prox", "qwap", "sage", "sams",
             "sany", "sch-", "sec-", "send", "seri", "sgh-", "shar", "sie-", "siem", "smal", "smar", "sony", "sph-",
-            "symb", "t-mo", "teli", "tim-", "tosh", "tsm-", "upg1", "upsi", "vk-v", "voda", "wap-", "wapa", "wapi",
+            "symb", "t-mo", "teli", "tim-",  "tsm-", "upg1", "upsi", "vk-v", "voda", "wap-", "wapa", "wapi",
             "wapp", "wapr", "webc", "winw", "winw", "xda", "xda-", "Googlebot-Mobile" };
 
     /**
      *
-     * @Title: JudgelsMobile @Description: TODO(判断是否是手机浏览器) @param @param
-     *         request @param @return 设定文件 @return boolean 返回类型 @throws
+     * 判断是否是手机浏览器
      */
     public static boolean isMobile(HttpServletRequest request) {
         boolean isMobile = false;
-        if (request.getHeader("User-Agent") != null) {
+        String userAgent = request.getHeader("User-Agent");
+        if (userAgent != null) {
+            userAgent = userAgent.toLowerCase();
             for (String mobileAgent : mobileAgents) {
-                if (request.getHeader("User-Agent").toLowerCase().indexOf(mobileAgent) > 0) {
+                if (userAgent.indexOf(mobileAgent) > 0) {
                     isMobile = true;
                     break;
                 }

novel-front/src/main/java/com/java2nb/novel/core/utils/Constants.java

@@ -12,7 +12,7 @@ public class Constants {
     public static final String TEMPLATE_PATH_PREFIX_KEY = "templatePathPrefixKey";
 
     /**
-     * 本地图片保存前缀
+     * 保存图片到自己的存储介质路径前缀
      * */
     public static final String LOCAL_PIC_PREFIX = "/localPic/";
 
@@ -30,4 +30,9 @@ public class Constants {
      * 首页设置的小说数量
      * */
     public static final int INDEX_BOOK_SETTING_NUM = 32;
+
+    /**
+     * 累积的最大点击量
+     * */
+    public static final Integer ADD_MAX_VISIT_COUNT = 10;
 }

novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java

@@ -5,6 +5,7 @@ import com.java2nb.novel.search.BookSP;
 import com.java2nb.novel.vo.BookVO;
 import org.apache.ibatis.annotations.Param;
 
+import java.util.Date;
 import java.util.List;
 
 /**
@@ -15,13 +16,13 @@ public interface FrontBookMapper extends BookMapper {
 
     List<BookVO> searchByPage(BookSP params);
 
-    void addVisitCount(@Param("bookId") Long bookId);
+    void addVisitCount(@Param("bookId") Long bookId, @Param("visitCount") Integer visitCount);
 
     List<Book> listRecBookByCatId(@Param("catId") Integer catId);
 
     void addCommentCount(@Param("bookId") Long bookId);
 
-    List<Book> queryNetworkPicBooks(@Param("limit") Integer limit,@Param("offset") Integer offset);
+    List<Book> queryNetworkPicBooks(@Param("localPicPrefix") String localPicPrefix, @Param("limit") Integer limit);
 
     /**
      * 按评分随机查询小说集合

novel-front/src/main/java/com/java2nb/novel/service/BookService.java

@@ -1,12 +1,14 @@
 package com.java2nb.novel.service;
 
 
+import com.github.pagehelper.PageInfo;
 import com.java2nb.novel.search.BookSP;
 import com.java2nb.novel.vo.BookCommentVO;
 import com.java2nb.novel.vo.BookSettingVO;
 import com.java2nb.novel.entity.*;
 import com.java2nb.novel.vo.BookVO;
 
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
@@ -44,9 +46,9 @@ public interface BookService {
      * @param params 搜索参数
      * @param page 页码
      * @param pageSize 分页大小
-     * @return 小说集合
+     * @return 小说集合分页信息
      * */
-    List<BookVO> searchByPage(BookSP params, int page, int pageSize);
+    PageInfo searchByPage(BookSP params, int page, int pageSize);
 
     /**
      * 查询小说分类列表
@@ -110,8 +112,8 @@ public interface BookService {
     /**
      * 增加点击次数
      * @param bookId 书籍ID
-     * */
-    void addVisitCount(Long bookId);
+     * @param visitCount*/
+    void addVisitCount(Long bookId, Integer visitCount);
 
     /**
      * 查询章节数
@@ -178,16 +180,17 @@ public interface BookService {
 
     /**
      * 查询网络图片的小说
+     *
+     * @param localPicPrefix
      * @param limit 查询条数
-     * @param offset 开始行数
      * @return 返回小说集合
      * */
-    List<Book> queryNetworkPicBooks(Integer limit, Integer offset);
+    List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit);
 
 
     /**
-     * 更新小说网络图片到本地
-     * @param picUrl 网络图片路径
+     * 更新爬取的小说网络图片到自己的存储介质（本地、OSS、fastDfs）
+     * @param picUrl 爬取的网络图片路径
      * @param bookId 小说ID
      */
     void updateBookPicToLocal(String picUrl, Long bookId);
@@ -197,6 +200,7 @@ public interface BookService {
      * @param userId 用户ID
      * @param page 页码
      * @param pageSize 分页大小
+     * @return 小说集合
      * */
     List<Book> listBookPageByUserId(Long userId, int page, int pageSize);
 
@@ -224,4 +228,13 @@ public interface BookService {
      * @param authorId 作者ID
      * */
     void addBookContent(Long bookId, String indexName, String content, Long authorId);
+
+
+    /**
+     * 根据更新时间分页查询书籍列表
+     * @param startDate 开始时间，包括该时间
+     * @param limit 查询数量
+     * @return 书籍列表
+     * */
+    List<Book> queryBookByUpdateTimeByPage(Date startDate, int limit);
 }

novel-front/src/main/java/com/java2nb/novel/service/FileService.java

@@ -0,0 +1,17 @@
+package com.java2nb.novel.service;
+
+
+/**
+ * @author 11797
+ */
+public interface FileService {
+
+    /**
+     * 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）
+     * @param picSrc 爬取的网络图片路径
+     * @param picSavePath 保存路径
+     * @return 新图片地址
+     * */
+    String transFile(String picSrc, String picSavePath);
+
+}

novel-front/src/main/java/com/java2nb/novel/service/SearchService.java

@@ -0,0 +1,27 @@
+package com.java2nb.novel.service;
+
+
+import com.github.pagehelper.PageInfo;
+import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.search.BookSP;
+
+/**
+ * @author 11797
+ */
+public interface SearchService {
+
+    /**
+     * 导入到es
+     * @param book 小说数据
+     */
+    void importToEs(Book book);
+
+    /**
+     * 搜索
+     * @param params 搜索参数
+     * @param page 当前页码
+     * @param pageSize 每页大小
+     * @return 分页信息
+     */
+    PageInfo searchBook(BookSP params, int page, int pageSize);
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java

@@ -2,25 +2,26 @@ package com.java2nb.novel.service.impl;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.pagehelper.PageHelper;
+import com.github.pagehelper.PageInfo;
 import com.java2nb.novel.core.cache.CacheKey;
 import com.java2nb.novel.core.cache.CacheService;
 import com.java2nb.novel.core.enums.ResponseStatus;
 import com.java2nb.novel.core.exception.BusinessException;
-import com.java2nb.novel.core.utils.BeanUtil;
-import com.java2nb.novel.core.utils.Constants;
-import com.java2nb.novel.core.utils.FileUtil;
-import com.java2nb.novel.core.utils.IdWorker;
+import com.java2nb.novel.core.utils.*;
 import com.java2nb.novel.entity.*;
 import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.mapper.*;
 import com.java2nb.novel.search.BookSP;
 import com.java2nb.novel.service.AuthorService;
 import com.java2nb.novel.service.BookService;
+import com.java2nb.novel.service.FileService;
+import com.java2nb.novel.service.SearchService;
 import com.java2nb.novel.vo.BookCommentVO;
 import com.java2nb.novel.vo.BookSettingVO;
 import com.java2nb.novel.vo.BookVO;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.mybatis.dynamic.sql.SortSpecification;
 import org.mybatis.dynamic.sql.render.RenderingStrategies;
@@ -50,14 +51,18 @@ import static org.mybatis.dynamic.sql.select.SelectDSL.select;
  */
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class BookServiceImpl implements BookService {
 
     /**
      * 本地图片保存路径
-     * */
+     */
     @Value("${pic.save.path}")
     private String picSavePath;
 
+    @Value("${spring.elasticsearch.enable}")
+    private Integer esEnable;
+
     private final FrontBookSettingMapper bookSettingMapper;
 
     private final FrontBookMapper bookMapper;
@@ -76,6 +81,10 @@ public class BookServiceImpl implements BookService {
 
     private final AuthorService authorService;
 
+    private final SearchService searchService;
+
+    private final FileService fileService;
+
 
     @SneakyThrows
     @Override
@@ -83,24 +92,24 @@ public class BookServiceImpl implements BookService {
         String result = cacheService.get(CacheKey.INDEX_BOOK_SETTINGS_KEY);
         if (result == null || result.length() < Constants.OBJECT_JSON_CACHE_EXIST_LENGTH) {
             List<BookSettingVO> list = bookSettingMapper.listVO();
-            if(list.size() == 0) {
+            if (list.size() == 0) {
                 //如果首页小说没有被设置，则初始化首页小说设置
                 list = initIndexBookSetting();
             }
             result = new ObjectMapper().writeValueAsString(list.stream().collect(Collectors.groupingBy(BookSettingVO::getType)));
             cacheService.set(CacheKey.INDEX_BOOK_SETTINGS_KEY, result);
         }
-        return new ObjectMapper().readValue(result,Map.class);
+        return new ObjectMapper().readValue(result, Map.class);
     }
 
 
     /**
      * 初始化首页小说设置
-     * */
+     */
     private List<BookSettingVO> initIndexBookSetting() {
         Date currentDate = new Date();
-        List<Book> books =  bookMapper.selectIdsByScoreAndRandom(Constants.INDEX_BOOK_SETTING_NUM);
-        if(books.size() == Constants.INDEX_BOOK_SETTING_NUM) {
+        List<Book> books = bookMapper.selectIdsByScoreAndRandom(Constants.INDEX_BOOK_SETTING_NUM);
+        if (books.size() == Constants.INDEX_BOOK_SETTING_NUM) {
             List<BookSetting> bookSettingList = new ArrayList<>(Constants.INDEX_BOOK_SETTING_NUM);
             List<BookSettingVO> bookSettingVOList = new ArrayList<>(Constants.INDEX_BOOK_SETTING_NUM);
             for (int i = 0; i < books.size(); i++) {
@@ -114,7 +123,7 @@ public class BookServiceImpl implements BookService {
                     type = 2;
                 } else if (i < 26) {
                     type = 3;
-                }else{
+                } else {
                     type = 4;
                 }
                 BookSettingVO bookSettingVO = new BookSettingVO();
@@ -126,8 +135,8 @@ public class BookServiceImpl implements BookService {
                 bookSetting.setUpdateTime(currentDate);
                 bookSettingList.add(bookSetting);
 
-                BeanUtils.copyProperties(book,bookSettingVO);
-                BeanUtils.copyProperties(bookSetting,bookSettingVO);
+                BeanUtils.copyProperties(book, bookSettingVO);
+                BeanUtils.copyProperties(bookSetting, bookSettingVO);
                 bookSettingVOList.add(bookSettingVO);
             }
 
@@ -164,25 +173,43 @@ public class BookServiceImpl implements BookService {
         List<BookVO> result = (List<BookVO>) cacheService.getObject(CacheKey.INDEX_UPDATE_BOOK_KEY);
         if (result == null || result.size() == 0) {
             List<Book> bookPOList = listRank((byte) 2, 23);
-            result = BeanUtil.copyList(bookPOList,BookVO.class);
+            result = BeanUtil.copyList(bookPOList, BookVO.class);
             cacheService.setObject(CacheKey.INDEX_UPDATE_BOOK_KEY, result, 60 * 10);
         }
         return result;
     }
 
     @Override
-    public List<BookVO> searchByPage(BookSP params, int page, int pageSize) {
-        PageHelper.startPage(page, pageSize);
+    public PageInfo searchByPage(BookSP params, int page, int pageSize) {
+
+
         if (params.getUpdatePeriod() != null) {
             long cur = System.currentTimeMillis();
             long period = params.getUpdatePeriod() * 24 * 3600 * 1000;
             long time = cur - period;
             params.setUpdateTimeMin(new Date(time));
         }
+
+        if (esEnable == 1) {
+
+
+            try {
+                return searchService.searchBook(params,page,pageSize);
+
+            }catch (Exception e){
+                log.error(e.getMessage(),e);
+            }
+
+
+        }
+        PageHelper.startPage(page, pageSize);
+
         if (StringUtils.isNotBlank(params.getSort())) {
             OrderByHelper.orderBy(params.getSort() + " desc");
         }
-        return bookMapper.searchByPage(params);
+        return new PageInfo<>(bookMapper.searchByPage(params));
+
+
     }
 
     @Override
@@ -197,7 +224,7 @@ public class BookServiceImpl implements BookService {
 
     @Override
     public Book queryBookDetail(Long bookId) {
-        SelectStatementProvider selectStatement = select(id, catName, catId, picUrl, bookName, authorId, authorName, bookDesc, bookStatus, visitCount, wordCount, lastIndexId, lastIndexName, lastIndexUpdateTime,score,status)
+        SelectStatementProvider selectStatement = select(book.allColumns())
                 .from(book)
                 .where(id, isEqualTo(bookId))
                 .build()
@@ -206,15 +233,15 @@ public class BookServiceImpl implements BookService {
     }
 
     @Override
-    public List<BookIndex> queryIndexList(Long bookId,String orderBy, Integer limit) {
-        if(StringUtils.isNotBlank(orderBy)){
+    public List<BookIndex> queryIndexList(Long bookId, String orderBy, Integer limit) {
+        if (StringUtils.isNotBlank(orderBy)) {
             OrderByHelper.orderBy(orderBy);
         }
-        if(limit != null){
-            PageHelper.startPage(1,limit);
+        if (limit != null) {
+            PageHelper.startPage(1, limit);
         }
 
-        SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.bookId, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.updateTime,BookIndexDynamicSqlSupport.isVip)
+        SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.bookId, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.updateTime, BookIndexDynamicSqlSupport.isVip)
                 .from(bookIndex)
                 .where(BookIndexDynamicSqlSupport.bookId, isEqualTo(bookId))
                 .build()
@@ -225,7 +252,7 @@ public class BookServiceImpl implements BookService {
 
     @Override
     public BookIndex queryBookIndex(Long bookIndexId) {
-        SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.bookId, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount, BookIndexDynamicSqlSupport.updateTime,BookIndexDynamicSqlSupport.isVip)
+        SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.bookId, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount, BookIndexDynamicSqlSupport.updateTime, BookIndexDynamicSqlSupport.isVip)
                 .from(bookIndex)
                 .where(BookIndexDynamicSqlSupport.id, isEqualTo(bookIndexId))
                 .build()
@@ -271,7 +298,7 @@ public class BookServiceImpl implements BookService {
 
     @Override
     public BookContent queryBookContent(Long bookIndexId) {
-        SelectStatementProvider selectStatement = select(BookContentDynamicSqlSupport.id,BookContentDynamicSqlSupport.content)
+        SelectStatementProvider selectStatement = select(BookContentDynamicSqlSupport.id, BookContentDynamicSqlSupport.content)
                 .from(bookContent)
                 .where(BookContentDynamicSqlSupport.indexId, isEqualTo(bookIndexId))
                 .limit(1)
@@ -305,7 +332,7 @@ public class BookServiceImpl implements BookService {
         }
         SelectStatementProvider selectStatement = select(id, catId, catName, bookName, lastIndexId, lastIndexName, authorId, authorName, picUrl, bookDesc, wordCount, lastIndexUpdateTime)
                 .from(book)
-                .where(wordCount,isGreaterThan(0))
+                .where(wordCount, isGreaterThan(0))
                 .orderBy(sortSpecification)
                 .limit(limit)
                 .build()
@@ -315,9 +342,8 @@ public class BookServiceImpl implements BookService {
     }
 
     @Override
-    public void addVisitCount(Long bookId) {
-        bookMapper.addVisitCount(bookId);
-
+    public void addVisitCount(Long bookId, Integer visitCount) {
+        bookMapper.addVisitCount(bookId,visitCount);
     }
 
     @Override
@@ -349,10 +375,10 @@ public class BookServiceImpl implements BookService {
     }
 
     @Override
-    public List<BookCommentVO> listCommentByPage(Long userId,Long bookId, int page, int pageSize) {
+    public List<BookCommentVO> listCommentByPage(Long userId, Long bookId, int page, int pageSize) {
         PageHelper.startPage(page, pageSize);
         OrderByHelper.orderBy("t1.create_time desc");
-        return bookCommentMapper.listCommentByPage(userId,bookId);
+        return bookCommentMapper.listCommentByPage(userId, bookId);
     }
 
     @Transactional(rollbackFor = Exception.class)
@@ -361,11 +387,11 @@ public class BookServiceImpl implements BookService {
         //判断该用户是否已评论过该书籍
         SelectStatementProvider selectStatement = select(count(BookCommentDynamicSqlSupport.id))
                 .from(bookComment)
-                .where(BookCommentDynamicSqlSupport.createUserId,isEqualTo(userId))
-                .and(BookCommentDynamicSqlSupport.bookId,isEqualTo(comment.getBookId()))
+                .where(BookCommentDynamicSqlSupport.createUserId, isEqualTo(userId))
+                .and(BookCommentDynamicSqlSupport.bookId, isEqualTo(comment.getBookId()))
                 .build()
                 .render(RenderingStrategies.MYBATIS3);
-        if(bookCommentMapper.count(selectStatement)>0){
+        if (bookCommentMapper.count(selectStatement) > 0) {
             throw new BusinessException(ResponseStatus.HAS_COMMENTS);
         }
         //增加评论
@@ -374,7 +400,7 @@ public class BookServiceImpl implements BookService {
         bookCommentMapper.insertSelective(comment);
         //增加书籍评论数
         bookMapper.addCommentCount(comment.getBookId());
-        
+
     }
 
     @Override
@@ -382,14 +408,14 @@ public class BookServiceImpl implements BookService {
         Long authorId;
         SelectStatementProvider selectStatement = select(BookAuthorDynamicSqlSupport.id)
                 .from(BookAuthorDynamicSqlSupport.bookAuthor)
-                .where(BookAuthorDynamicSqlSupport.penName,isEqualTo(authorName))
+                .where(BookAuthorDynamicSqlSupport.penName, isEqualTo(authorName))
                 .build()
                 .render(RenderingStrategies.MYBATIS3);
         List<BookAuthor> bookAuthors = bookAuthorMapper.selectMany(selectStatement);
-        if(bookAuthors.size()>0){
+        if (bookAuthors.size() > 0) {
             //作者存在
             authorId = bookAuthors.get(0).getId();
-        }else{
+        } else {
             //作者不存在，先创建作者
             Date currentDate = new Date();
             authorId = new IdWorker().nextId();
@@ -409,18 +435,17 @@ public class BookServiceImpl implements BookService {
     }
 
 
-
     @Override
     public Long queryIdByNameAndAuthor(String bookName, String author) {
         //查询小说ID
         SelectStatementProvider selectStatement = select(id)
                 .from(book)
-                .where(BookDynamicSqlSupport.bookName,isEqualTo(bookName))
-                .and(BookDynamicSqlSupport.authorName,isEqualTo(authorName))
+                .where(BookDynamicSqlSupport.bookName, isEqualTo(bookName))
+                .and(BookDynamicSqlSupport.authorName, isEqualTo(authorName))
                 .build()
                 .render(RenderingStrategies.MYBATIS3);
         List<Book> books = bookMapper.selectMany(selectStatement);
-        if(books.size()>0){
+        if (books.size() > 0) {
             return books.get(0).getId();
         }
         return null;
@@ -430,7 +455,7 @@ public class BookServiceImpl implements BookService {
     public List<Integer> queryIndexNumByBookId(Long bookId) {
         SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.indexNum)
                 .from(BookIndexDynamicSqlSupport.bookIndex)
-                .where(BookIndexDynamicSqlSupport.bookId,isEqualTo(bookId))
+                .where(BookIndexDynamicSqlSupport.bookId, isEqualTo(bookId))
                 .build()
                 .render(RenderingStrategies.MYBATIS3);
 
@@ -438,21 +463,21 @@ public class BookServiceImpl implements BookService {
     }
 
     @Override
-    public List<Book> queryNetworkPicBooks(Integer limit, Integer offset) {
-        return bookMapper.queryNetworkPicBooks(limit,offset);
+    public List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit) {
+        return bookMapper.queryNetworkPicBooks(localPicPrefix,limit);
     }
 
     @Override
     public void updateBookPicToLocal(String picUrl, Long bookId) {
 
-        picUrl = FileUtil.network2Local(picUrl,picSavePath, Constants.LOCAL_PIC_PREFIX);
+        picUrl = fileService.transFile(picUrl, picSavePath);
 
         bookMapper.update(update(book)
                 .set(BookDynamicSqlSupport.picUrl)
                 .equalTo(picUrl)
                 .set(updateTime)
                 .equalTo(new Date())
-                .where(id,isEqualTo(bookId))
+                .where(id, isEqualTo(bookId))
                 .build()
                 .render(RenderingStrategies.MYBATIS3));
 
@@ -461,7 +486,7 @@ public class BookServiceImpl implements BookService {
     @Override
     public List<Book> listBookPageByUserId(Long userId, int page, int pageSize) {
 
-        PageHelper.startPage(page,pageSize);
+        PageHelper.startPage(page, pageSize);
 
         SelectStatementProvider selectStatement = select(id, bookName, visitCount, lastIndexName, status)
                 .from(book)
@@ -476,10 +501,11 @@ public class BookServiceImpl implements BookService {
     @Override
     public void addBook(Book book, Long authorId, String penName) {
         //判断小说名是否存在
-        if(queryIdByNameAndAuthor(book.getBookName(),penName)!=null){
+        if (queryIdByNameAndAuthor(book.getBookName(), penName) != null) {
             //该作者发布过此书名的小说
             throw new BusinessException(ResponseStatus.BOOKNAME_EXISTS);
-        };
+        }
+        ;
         book.setAuthorName(penName);
         book.setAuthorId(authorId);
         book.setVisitCount(0L);
@@ -497,18 +523,18 @@ public class BookServiceImpl implements BookService {
         bookMapper.update(update(book)
                 .set(BookDynamicSqlSupport.status)
                 .equalTo(status)
-                .where(id,isEqualTo(bookId))
-                .and(BookDynamicSqlSupport.authorId,isEqualTo(authorId))
+                .where(id, isEqualTo(bookId))
+                .and(BookDynamicSqlSupport.authorId, isEqualTo(authorId))
                 .build()
                 .render(RenderingStrategies.MYBATIS3));
     }
 
-    @Transactional
+    @Transactional(rollbackFor = Exception.class)
     @Override
     public void addBookContent(Long bookId, String indexName, String content, Long authorId) {
 
         Book book = queryBookDetail(bookId);
-        if(!authorId.equals(book.getAuthorId())){
+        if (!authorId.equals(book.getAuthorId())) {
             //并不是更新自己的小说
             return;
         }
@@ -525,15 +551,15 @@ public class BookServiceImpl implements BookService {
                 .set(BookDynamicSqlSupport.lastIndexUpdateTime)
                 .equalTo(currentDate)
                 .set(BookDynamicSqlSupport.wordCount)
-                .equalTo(book.getWordCount()+wordCount)
-                .where(id,isEqualTo(bookId))
-                .and(BookDynamicSqlSupport.authorId,isEqualTo(authorId))
+                .equalTo(book.getWordCount() + wordCount)
+                .where(id, isEqualTo(bookId))
+                .and(BookDynamicSqlSupport.authorId, isEqualTo(authorId))
                 .build()
                 .render(RenderingStrategies.MYBATIS3));
         //更新小说目录表
         int indexNum = 0;
-        if(book.getLastIndexId() != null){
-            indexNum =  queryBookIndex(book.getLastIndexId()).getIndexNum()+1;
+        if (book.getLastIndexId() != null) {
+            indexNum = queryBookIndex(book.getLastIndexId()).getIndexNum() + 1;
         }
         BookIndex lastBookIndex = new BookIndex();
         lastBookIndex.setId(lastIndexId);
@@ -553,7 +579,19 @@ public class BookServiceImpl implements BookService {
         bookContentMapper.insertSelective(bookContent);
 
 
+    }
 
+    @Override
+    public List<Book> queryBookByUpdateTimeByPage(Date startDate,  int limit) {
+
+
+        return bookMapper.selectMany(select(book.allColumns())
+                .from(book)
+                .where(updateTime, isGreaterThan(startDate))
+                .orderBy(updateTime)
+                .limit(limit)
+                .build()
+                .render(RenderingStrategies.MYBATIS3));
     }
 
 

novel-front/src/main/java/com/java2nb/novel/service/impl/FastDfsFileServiceImpl.java

@@ -0,0 +1,60 @@
+package com.java2nb.novel.service.impl;
+
+import com.github.tobato.fastdfs.domain.StorePath;
+import com.github.tobato.fastdfs.service.FastFileStorageClient;
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FilenameUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.io.FileInputStream;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@Slf4j
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "fastDfs")
+public class FastDfsFileServiceImpl implements FileService {
+
+    private final FastFileStorageClient storageClient;
+
+    @Value("${fdfs.webUrl}")
+    private String webUrl;
+
+    @Override
+    public String transFile(String picSrc, String picSavePath) {
+
+        File file;
+        String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+        if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
+            file = new File(picSavePath + filePath);
+        } else {
+            //默认图片不存储
+            return filePath;
+        }
+
+        try {
+            FileInputStream inputStream = new FileInputStream(file);
+            StorePath storePath = storageClient.uploadFile(inputStream, file.length(),
+                    FilenameUtils.getExtension(file.getName()), null);
+            //这里额外加上LOCAL_PIC_PREFIX路径，表明该图片是个人资源，而不是爬虫爬取的网络资源，不需要再次进行转换，
+            // 实际访问时，再通过nginx的rewite指令来重写路径，去掉LOCAL_PIC_PREFIX
+            return webUrl+Constants.LOCAL_PIC_PREFIX+storePath.getFullPath();
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        } finally {
+            //删除
+            file.delete();
+        }
+
+        return "/images/default.gif";
+    }
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/FriendLinkServiceImpl.java

@@ -40,7 +40,7 @@ public class FriendLinkServiceImpl implements FriendLinkService {
                     .build()
                     .render(RenderingStrategies.MYBATIS3);
             result =  friendLinkMapper.selectMany(selectStatement);
-            cacheService.setObject(CacheKey.INDEX_LINK_KEY,result);
+            cacheService.setObject(CacheKey.INDEX_LINK_KEY,result,60 * 60 * 24);
         }
         return result;
     }

novel-front/src/main/java/com/java2nb/novel/service/impl/LocalFileServiceImpl.java

@@ -0,0 +1,23 @@
+package com.java2nb.novel.service.impl;
+
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "local")
+public class LocalFileServiceImpl implements FileService {
+
+    @Override
+    public String transFile(String picSrc, String picSavePath){
+
+        return FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+    }
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/NewsServiceImpl.java

@@ -42,7 +42,7 @@ public class NewsServiceImpl implements NewsService {
                     .build()
                     .render(RenderingStrategies.MYBATIS3);
             result = newsMapper.selectMany(selectStatement);
-            cacheService.setObject(CacheKey.INDEX_NEWS_KEY,result);
+            cacheService.setObject(CacheKey.INDEX_NEWS_KEY,result,60 * 60 * 12);
         }
         return result;
     }

novel-front/src/main/java/com/java2nb/novel/service/impl/OssFileServiceImpl.java

@@ -0,0 +1,76 @@
+package com.java2nb.novel.service.impl;
+
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.model.CannedAccessControlList;
+import com.aliyun.oss.model.CreateBucketRequest;
+import com.aliyun.oss.model.PutObjectRequest;
+import com.aliyun.oss.model.PutObjectResult;
+import com.java2nb.novel.core.config.OssProperties;
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "OSS")
+@Slf4j
+public class OssFileServiceImpl implements FileService {
+
+    private final OssProperties ossProperties;
+
+    @Override
+    public String transFile(String picSrc, String picSavePath) {
+
+        File file;
+        String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+        if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
+            file = new File(picSavePath+filePath);
+        } else {
+            //默认图片不存储
+            return filePath;
+        }
+
+        filePath = filePath.replaceFirst(picSavePath,"");
+
+        filePath = filePath.startsWith("/") ? filePath.replaceFirst("/","") : filePath;
+
+
+        OSSClient ossClient = new OSSClient(ossProperties.getEndpoint(), ossProperties.getKeyId(), ossProperties.getKeySecret());
+        try {
+            //容器不存在，就创建
+            if (!ossClient.doesBucketExist(ossProperties.getBucketName())) {
+                ossClient.createBucket(ossProperties.getBucketName());
+                CreateBucketRequest createBucketRequest = new CreateBucketRequest(ossProperties.getBucketName());
+                createBucketRequest.setCannedACL(CannedAccessControlList.PublicRead);
+                ossClient.createBucket(createBucketRequest);
+            }
+            //上传文件
+            PutObjectResult result = ossClient.putObject(new PutObjectRequest(ossProperties.getBucketName(), filePath, file));
+            //设置权限 这里是公开读
+            ossClient.setBucketAcl(ossProperties.getBucketName(), CannedAccessControlList.PublicRead);
+
+            if(result != null) {
+                return ossProperties.getWebUrl() + "/" + filePath;
+            }
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        } finally {
+            //关闭
+            ossClient.shutdown();
+            file.delete();
+        }
+
+        return "/images/default.gif";
+    }
+
+
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/SearchServiceImpl.java

@@ -0,0 +1,202 @@
+package com.java2nb.novel.service.impl;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.pagehelper.PageInfo;
+import com.java2nb.novel.core.enums.ResponseStatus;
+import com.java2nb.novel.core.exception.BusinessException;
+import com.java2nb.novel.core.utils.StringUtil;
+import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.search.BookSP;
+import com.java2nb.novel.service.SearchService;
+import com.java2nb.novel.vo.EsBookVO;
+import io.searchbox.client.JestClient;
+import io.searchbox.core.*;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.springframework.beans.BeanUtils;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class SearchServiceImpl implements SearchService {
+
+    private final String INDEX = "novel";
+
+    private final String TYPE = "book";
+
+    private final JestClient jestClient;
+
+
+    @Override
+    @SneakyThrows
+    public void importToEs(Book book) {
+        //导入到ES
+        EsBookVO esBookVO = new EsBookVO();
+        BeanUtils.copyProperties(book, esBookVO, "lastIndexUpdateTime");
+        esBookVO.setLastIndexUpdateTime(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(book.getLastIndexUpdateTime()));
+        Index action = new Index.Builder(esBookVO).index(INDEX).type(TYPE).id(book.getId().toString()).build();
+
+        jestClient.execute(action);
+
+
+    }
+
+    @SneakyThrows
+    @Override
+    public PageInfo searchBook(BookSP params, int page, int pageSize) {
+        List<EsBookVO> bookList = new ArrayList<>(0);
+
+        //使用搜索引擎搜索
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        // 构造查询哪个字段
+        if (StringUtils.isNoneBlank(params.getKeyword())) {
+            boolQueryBuilder = boolQueryBuilder.must(QueryBuilders.queryStringQuery(params.getKeyword()));
+        }
+
+        // 作品方向
+        if (params.getWorkDirection() != null) {
+            boolQueryBuilder.filter(QueryBuilders.termQuery("workDirection", params.getWorkDirection()));
+        }
+
+        // 分类
+        if (params.getCatId() != null) {
+            boolQueryBuilder.filter(QueryBuilders.termQuery("catId", params.getCatId()));
+        }
+        if (params.getBookStatus() != null) {
+            boolQueryBuilder.filter(QueryBuilders.termQuery("bookStatus", params.getBookStatus()));
+        }
+
+        if (params.getWordCountMin() == null) {
+            params.setWordCountMin(0);
+        }
+        if (params.getWordCountMax() == null) {
+            params.setWordCountMax(Integer.MAX_VALUE);
+        }
+
+        boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
+
+        if (params.getUpdateTimeMin() != null) {
+            boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(params.getUpdateTimeMin())));
+        }
+
+
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.query(boolQueryBuilder);
+
+
+        Count count = new Count.Builder().addIndex(INDEX).addType(TYPE)
+                .query(searchSourceBuilder.toString()).build();
+        CountResult results = jestClient.execute(count);
+        Double total = results.getCount();
+
+
+        // 高亮字段
+        HighlightBuilder highlightBuilder = new HighlightBuilder();
+        highlightBuilder.field("authorName");
+        highlightBuilder.field("bookName");
+        highlightBuilder.field("bookDesc");
+        highlightBuilder.field("lastIndexName");
+        highlightBuilder.field("catName");
+        highlightBuilder.preTags("<span style='color:red'>").postTags("</span>");
+        highlightBuilder.fragmentSize(20000);
+        searchSourceBuilder.highlighter(highlightBuilder);
+
+
+        //设置排序
+        if (params.getSort() != null) {
+            searchSourceBuilder.sort(StringUtil.camelName(params.getSort()), SortOrder.DESC);
+        }
+
+        // 设置分页
+        searchSourceBuilder.from((page - 1) * pageSize);
+        searchSourceBuilder.size(pageSize);
+
+        // 构建Search对象
+        Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(INDEX).addType(TYPE).build();
+        log.debug(search.toString());
+        SearchResult result;
+        result = jestClient.execute(search);
+        log.debug(result.getJsonString());
+        if (result.isSucceeded()) {
+
+            Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
+            if (resultMap.get("hits") != null) {
+                Map hitsMap = (Map) resultMap.get("hits");
+                if (hitsMap.size() > 0 && hitsMap.get("hits") != null) {
+                    List hitsList = (List) hitsMap.get("hits");
+                    if (hitsList.size() > 0 && result.getSourceAsString() != null) {
+
+                        JavaType jt = new ObjectMapper().getTypeFactory().constructParametricType(ArrayList.class, EsBookVO.class);
+                        bookList = new ObjectMapper().readValue("[" + result.getSourceAsString() + "]", jt);
+
+                        if (bookList != null) {
+                            for (int i = 0; i < bookList.size(); i++) {
+                                hitsMap = (Map) hitsList.get(i);
+                                Map highlightMap = (Map) hitsMap.get("highlight");
+                                if (highlightMap != null && highlightMap.size() > 0) {
+
+                                    List<String> authorNameList = (List<String>) highlightMap.get("authorName");
+                                    if (authorNameList != null && authorNameList.size() > 0) {
+                                        bookList.get(i).setAuthorName(authorNameList.get(0));
+                                    }
+
+                                    List<String> bookNameList = (List<String>) highlightMap.get("bookName");
+                                    if (bookNameList != null && bookNameList.size() > 0) {
+                                        bookList.get(i).setBookName(bookNameList.get(0));
+                                    }
+
+                                    List<String> bookDescList = (List<String>) highlightMap.get("bookDesc");
+                                    if (bookDescList != null && bookDescList.size() > 0) {
+                                        bookList.get(i).setBookDesc(bookDescList.get(0));
+                                    }
+
+                                    List<String> lastIndexNameList = (List<String>) highlightMap.get("lastIndexName");
+                                    if (lastIndexNameList != null && lastIndexNameList.size() > 0) {
+                                        bookList.get(i).setLastIndexName(lastIndexNameList.get(0));
+                                    }
+
+                                    List<String> catNameList = (List<String>) highlightMap.get("catName");
+                                    if (catNameList != null && catNameList.size() > 0) {
+                                        bookList.get(i).setCatName(catNameList.get(0));
+                                    }
+
+
+                                }
+                            }
+
+
+                        }
+                    }
+                }
+            }
+
+            PageInfo<EsBookVO> pageInfo = new PageInfo<>(bookList);
+            pageInfo.setTotal(total.longValue());
+            pageInfo.setPageNum(page);
+            pageInfo.setPageSize(pageSize);
+            return pageInfo;
+        }
+       throw new BusinessException(ResponseStatus.ES_SEARCH_FAIL);
+    }
+
+
+
+}

novel-front/src/main/java/com/java2nb/novel/vo/EsBookVO.java

@@ -0,0 +1,85 @@
+package com.java2nb.novel.vo;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.java2nb.novel.entity.Book;
+import lombok.Data;
+
+import javax.annotation.Generated;
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * @author Administrator
+ */
+@Data
+public class EsBookVO  {
+
+    private Long id;
+
+    
+    private Byte workDirection;
+
+    
+    private Integer catId;
+
+    
+    private String catName;
+
+    
+    private String picUrl;
+
+    
+    private String bookName;
+
+    
+    private Long authorId;
+
+    
+    private String authorName;
+
+    
+    private String bookDesc;
+
+    
+    private Float score;
+
+    
+    private Byte bookStatus;
+
+    
+    private Long visitCount;
+
+    
+    private Integer wordCount;
+
+    
+    private Integer commentCount;
+
+    
+    private Long lastIndexId;
+
+    
+    private String lastIndexName;
+
+    private String lastIndexUpdateTime;
+
+    
+    private Byte isVip;
+
+    
+    private Byte status;
+
+    
+
+    
+    private Integer crawlSourceId;
+
+    
+    private String crawlBookId;
+
+    
+
+    private Byte crawlIsStop;
+
+
+}

novel-front/src/main/resources/application-fastdfs.yml

@@ -0,0 +1,16 @@
+#FastDfs的配置	====================================
+#读取inputsream阻塞时间
+fdfs:
+  connect-timeout: 600
+  so-timeout: 1500
+  #tracker地址
+  trackerList: 127.0.0.1:22122
+  #缩略图配置
+  thumbImage:
+    height: 150
+    width: 150
+  #通过nginx 访问地址
+  webUrl: http://127.0.0.1/
+  #获取连接池最大数量
+  pool.max-total: 200
+

novel-front/src/main/resources/application-oss.yml

@@ -0,0 +1,9 @@
+#OSS
+novel:
+  file:
+    endpoint: oss-cn-shenzhen.aliyuncs.com #不同的服务器，地址不同
+    key-id: dhjjrue6767778878 #去OSS控制台获取
+    key-secret: dssdkkrkelrkellk44554 #去OSS控制台获取
+    bucket-name: novel #这个自己创建bucket时的命名，控制台创建也行，代码创建也行
+    file-host: pic #文件路径
+    web-url: https://xxyimg.oss-cn-hangzhou.aliyuncs.com #OSS文件的web访问地址

novel-front/src/main/resources/application.yml

@@ -4,7 +4,30 @@ server:
 spring:
   profiles:
     active: dev
-    include: alipay
+    include: alipay,oss,fastdfs
+
+
+  rabbitmq:
+    enable: 0
+    host: 127.0.0.1
+    username: guest
+    password: guest
+    virtual-host: /novel-plus
+    template:
+      # 缺省的交换机名称，此处配置后，发送消息如果不指定交换机就会使用这个
+      exchange: novel.exchange
+    publisher-confirms: false
+
+
+  elasticsearch:
+    #是否开启搜索引擎，1：开启，0：不开启
+    enable: 0
+    jest:
+      uris: http://127.0.0.1:9200
+
+redisson:
+  singleServerConfig:
+    address: 127.0.0.1:6379
 
 jwt:
   secret: novel!#20191230

novel-front/src/main/resources/mybatis/mapping/BookMapper.xml

@@ -34,8 +34,9 @@
 
     </select>
 
-    <update id="addVisitCount" parameterType="long">
-        update book set visit_count = visit_count + 1 where id = #{bookId}
+    <update id="addVisitCount" >
+        update book set visit_count = visit_count + ${visitCount}
+        where id = #{bookId}
     </update>
 
     <select id="listRecBookByCatId" parameterType="int" resultType="com.java2nb.novel.entity.Book">
@@ -54,8 +55,9 @@
     <select id="queryNetworkPicBooks" resultType="com.java2nb.novel.entity.Book">
         select
         id,pic_url from book
-        where pic_url like 'http://%' or pic_url like 'https://%'
-        limit #{offset},#{limit}
+        where pic_url like 'http%'
+        and pic_url not like concat('%',#{localPicPrefix},'%')
+        limit #{limit}
     </select>
 
     <select id="selectIdsByScoreAndRandom" parameterType="int" resultType="com.java2nb.novel.entity.Book">

novel-front/src/main/resources/static/javascript/ajaxfileupload.js

@@ -0,0 +1,217 @@
+jQuery.extend({
+    createUploadIframe: function(id, uri)
+    {
+        //create frame
+        var frameId = 'jUploadFrame' + id;
+        var iframeHtml = '<iframe id="' + frameId + '" name="' + frameId + '" style="position:absolute; top:-9999px; left:-9999px"';
+        if(window.ActiveXObject)
+        {
+            if(typeof uri== 'boolean'){
+                iframeHtml += ' src="' + 'javascript:false' + '"';
+
+            }
+            else if(typeof uri== 'string'){
+                iframeHtml += ' src="' + uri + '"';
+
+            }
+        }
+        iframeHtml += ' />';
+        jQuery(iframeHtml).appendTo(document.body);
+
+        return jQuery('#' + frameId).get(0);
+    },
+    createUploadForm: function(id,fileElementId,data,fileElement)
+    {
+        //create form
+        var formId = 'jUploadForm' + id;
+        var fileId = 'jUploadFile' + id;
+        var form = jQuery('<form  action="" method="POST" name="' + formId + '" id="' + formId + '" enctype="multipart/form-data"></form>');
+        if(data)
+        {
+            for(var i in data)
+            {
+                jQuery('<input type="hidden" name="' + i + '" value="' + data[i] + '" />').appendTo(form);
+            }
+        }
+        var oldElement;
+        if(fileElement == null)
+            oldElement = jQuery('#' + fileElementId);
+        else
+            oldElement = fileElement;
+
+        var newElement = jQuery(oldElement).clone();
+        jQuery(oldElement).attr('id', fileId);
+        jQuery(oldElement).before(newElement);
+        jQuery(oldElement).appendTo(form);
+
+        //set attributes
+        jQuery(form).css('position', 'absolute');
+        jQuery(form).css('top', '-1200px');
+        jQuery(form).css('left', '-1200px');
+        jQuery(form).appendTo('body');
+        return form;
+    },
+
+    ajaxFileUpload: function(s) {
+        // TODO introduce global settings, allowing the client to modify them for all requests, not only timeout
+        s = jQuery.extend({}, jQuery.ajaxSettings, s);
+        var id = new Date().getTime()
+        var form = jQuery.createUploadForm(id, s.fileElementId, (typeof(s.data)=='undefined'?false:s.data),s.fileElement);
+        var io = jQuery.createUploadIframe(id, s.secureuri);
+        var frameId = 'jUploadFrame' + id;
+        var formId = 'jUploadForm' + id;
+        // Watch for a new set of requests
+        if ( s.global && ! jQuery.active++ )
+        {
+            jQuery.event.trigger( "ajaxStart" );
+        }
+        var requestDone = false;
+        // Create the request object
+        var xml = {}
+        if ( s.global )
+            jQuery.event.trigger("ajaxSend", [xml, s]);
+        // Wait for a response to come back
+        var uploadCallback = function(isTimeout)
+        {
+            var io = document.getElementById(frameId);
+            try
+            {
+                if(io.contentWindow)
+                {
+                    xml.responseText = io.contentWindow.document.body?io.contentWindow.document.body.innerHTML:null;
+                    xml.responseXML = io.contentWindow.document.XMLDocument?io.contentWindow.document.XMLDocument:io.contentWindow.document;
+
+                }else if(io.contentDocument)
+                {
+                    xml.responseText = io.contentDocument.document.body?io.contentDocument.document.body.innerHTML:null;
+                    xml.responseXML = io.contentDocument.document.XMLDocument?io.contentDocument.document.XMLDocument:io.contentDocument.document;
+                }
+            }catch(e)
+            {
+                jQuery.handleError(s, xml, null, e);
+            }
+            if ( xml || isTimeout == "timeout")
+            {
+                requestDone = true;
+                var status;
+                try {
+                    status = isTimeout != "timeout" ? "success" : "error";
+                    // Make sure that the request was successful or notmodified
+                    if ( status != "error" )
+                    {
+                        // process the data (runs the xml through httpData regardless of callback)
+                        var data = jQuery.uploadHttpData( xml, s.dataType );
+                        // If a local callback was specified, fire it and pass it the data
+                        if ( s.success )
+                            s.success( data, status );
+
+                        // Fire the global callback
+                        if( s.global )
+                            jQuery.event.trigger( "ajaxSuccess", [xml, s] );
+                    } else
+                        jQuery.handleError(s, xml, status);
+                } catch(e)
+                {
+                    status = "error";
+                    jQuery.handleError(s, xml, status, e);
+                }
+
+                // The request was completed
+                if( s.global )
+                    jQuery.event.trigger( "ajaxComplete", [xml, s] );
+
+                // Handle the global AJAX counter
+                if ( s.global && ! --jQuery.active )
+                    jQuery.event.trigger( "ajaxStop" );
+
+                // Process result
+                if ( s.complete )
+                    s.complete(xml, status);
+
+                jQuery(io).unbind()
+
+                setTimeout(function()
+                {	try
+                {
+                    jQuery(io).remove();
+                    jQuery(form).remove();
+
+                } catch(e)
+                {
+                    jQuery.handleError(s, xml, null, e);
+                }
+
+                }, 100)
+
+                xml = null
+
+            }
+        }
+        // Timeout checker
+        if ( s.timeout > 0 )
+        {
+            setTimeout(function(){
+                // Check to see if the request is still happening
+                if( !requestDone ) uploadCallback( "timeout" );
+            }, s.timeout);
+        }
+        try
+        {
+
+            var form = jQuery('#' + formId);
+            jQuery(form).attr('action', s.url);
+            jQuery(form).attr('method', 'POST');
+            jQuery(form).attr('target', frameId);
+            if(form.encoding)
+            {
+                jQuery(form).attr('encoding', 'multipart/form-data');
+            }
+            else
+            {
+                jQuery(form).attr('enctype', 'multipart/form-data');
+            }
+            jQuery(form).submit();
+
+        } catch(e)
+        {
+            jQuery.handleError(s, xml, null, e);
+        }
+
+        jQuery('#' + frameId).load(uploadCallback);
+        return {abort: function(){
+                try
+                {
+                    jQuery('#' + frameId).remove();
+                    jQuery(form).remove();
+                }
+                catch(e){}
+            }};
+    },
+
+    uploadHttpData: function( r, type ) {
+        var data = !type;
+        data = type == "xml" || data ? r.responseXML : r.responseText;
+
+        // If the type is "script", eval it in global context
+        if ( type == "script" )
+            jQuery.globalEval( data );
+        // Get the JavaScript object, if JSON is used.
+        if ( type == "json" )
+            data = jQuery.parseJSON(jQuery(data).text());
+        // evaluate scripts within html
+        if ( type == "html" )
+            jQuery("<div>").html(data).evalScripts();
+
+        return data;
+    },
+
+    handleError: function( s, xml, status, e ) {
+        // If a local callback was specified, fire it
+        if ( s.error )
+            s.error( xml, status, e );
+
+        // Fire the global callback
+        if ( s.global )
+            jQuery.event.trigger( "ajaxError", [xml, s, e] );
+    }
+});

novel-front/src/main/resources/templates/author/book_add.html

@@ -8,6 +8,9 @@
         <title>作家管理系统-小说精品屋</title>
         <link rel="stylesheet" href="/css/base.css?v=1"/>
         <link rel="stylesheet" href="/css/user.css"/>
+        <style type="text/css">
+            .opacity{-khtml-opacity:0.0;-moz-opacity:0.0;filter:alpha(opacity=0);filter:"alpha(opacity=0)";opacity:0.0; filter: progid:DXImageTransform.Microsoft.Alpha(opacity=0);/*兼容ie8及以下*/}
+        </style>
     </head>
 </head>
 <body class="">
@@ -74,8 +77,15 @@
                                 <input type="hidden" id="catName" name="catName" value="玄幻奇幻"/>
                                 <b>小说名：</b>
                                 <li><input type="text" id="bookName" name="bookName" class="s_input" ></li>
-                                <b>小说封面图片路径：</b>
-                                <li><input type="text" id="picUrl" name="picUrl" class="s_input" ></li>
+                                <b>小说封面：</b>
+                                <li style="position: relative">
+                                    <input class="opacity" onchange="picChange()"
+                                           type="file" id="file0" name="file"
+                                           title="点击更换图片"
+                                           style="z-index: 100;cursor: pointer;left: 0px; top: 0px; width: 100px; height: 130px; opacity: 0; position: absolute;  "
+                                    />
+                                    <img style="width:100px;height: 130px" id="picImage" src="/images/default.gif" alt="">
+                                    <input type="hidden" id="picUrl" name="picUrl" class="s_input" value="/images/default.gif"></li>
                                 <b>小说介绍：</b>
 
                                 <li><textarea name="bookDesc" rows="5" cols="53" id="bookDesc"
@@ -126,37 +136,78 @@
 
 </body>
 <script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
+<script src="/javascript/ajaxfileupload.js" type="text/javascript"></script>
+
 <script src="/layui/layui.all.js" type="text/javascript"></script>
 <script src="/javascript/header.js" type="text/javascript"></script>
 <script src="/javascript/user.js" type="text/javascript"></script>
 <script language="javascript" type="text/javascript">
+
+    function picChange() {
+        var file = $("#file0").val(); //文件名称
+        if (file != "") {
+
+            $.ajaxFileUpload({
+                url : "/file/upload", //用于文件上传的服务器端请求地址
+                secureuri : false, //是否需要安全协议，一般设置为false
+                fileElementId : "file0", //文件上传域的ID
+                dataType : "json", //返回值类型 一般设置为json
+                type : "post",
+                success : function(data) { //服务器成功响应处理函数
+                    if (data.code == 200) {
+                        $("#picImage").attr("src", data.data);
+                        $("#picUrl").val(data.data);
+                    }else {
+                        layer.alert('图片上传失败');
+                    }
+
+                }
+
+            });
+        } else {
+            alert("请选择上传文件！");
+        }
+
+
+
+    }
+
+    var lock = false;
     function addBook() {
+        if(lock){
+            return;
+        }
+        lock = true;
+
 
 
         var bookName = $("#bookName").val();
         if(!bookName){
             $("#LabErr").html("小说名不能为空！");
+            lock = false;
             return;
         }
         if(bookName.length > 20){
             $("#LabErr").html("小说名太长！");
+            lock = false;
             return;
         }
 
         var picUrl = $("#picUrl").val();
         if(!picUrl){
             $("#LabErr").html("封面图片不能为空！");
+            lock = false;
             return;
         }
 
         var bookDesc = $("#bookDesc").val();
         if(!bookDesc){
             $("#LabErr").html("简介不能为空！");
+            lock = false;
             return;
         }
 
 
-
         $.ajax({
             type: "POST",
             url: "/author/addBook",
@@ -171,11 +222,13 @@
 
 
                 } else {
+                    lock = false;
                     $("#LabErr").html(data.msg);
                 }
 
             },
             error: function () {
+                lock = false;
                 layer.alert('网络异常');
             }
         })

novel-front/src/main/resources/templates/author/content_add.html

@@ -58,10 +58,8 @@
                                 <b>章节名：</b>
                                 <li><input type="text" id="bookIndex" name="bookIndex" class="s_input" ></li>
                                 <b>章节内容：</b>
-                                <input type="hidden" id="bookContent" name="bookContent"  >
-                                <div style="width:600px" id="contentEditor">
-
-                                </div>
+                                <textarea name="bookContent" rows="30" cols="80" id="bookContent"
+                                          class="textarea"></textarea>
                                 <li style="margin-top: 10px"><input type="button" onclick="addBookContent()" name="btnRegister" value="提交"
                                            id="btnRegister" class="btn_red">
 
@@ -114,43 +112,18 @@
 <script src="/javascript/header.js" type="text/javascript"></script>
 <script src="/javascript/user.js" type="text/javascript"></script>
 <script src="/javascript/common.js" type="text/javascript"></script>
-<script type="text/javascript" src="/wangEditor/release/wangEditor.js"></script>
 
 <script language="javascript" type="text/javascript">
 
-    var E = window.wangEditor;
-    var editor2 = new E('#contentEditor');
-    // 自定义菜单配置
-    editor2.customConfig.menus = [
-        'head',  // 标题
-        'bold',  // 粗体
-        'fontSize',  // 字号
-        'fontName',  // 字体
-        'italic',  // 斜体
-        'underline',  // 下划线
-        'strikeThrough',  // 删除线
-        'foreColor',  // 文字颜色
-        //'backColor',  // 背景颜色
-        //'link',  // 插入链接
-        'list',  // 列表
-        'justify',  // 对齐方式
-        'quote',  // 引用
-        'emoticon',  // 表情
-        'image',  // 插入图片
-        //'table',  // 表格
-        //'video',  // 插入视频
-        //'code',  // 插入代码
-        'undo',  // 撤销
-        'redo'  // 重复
-    ];
-    editor2.customConfig.onchange = function (html) {
-        // html 即变化之后的内容
-        $("#bookContent").val(html);
-    }
-    editor2.create();
 
+    var lock = false;
     function addBookContent() {
 
+        if(lock){
+            return;
+        }
+        lock = true;
+
 
         var bookId = getSearchString("bookId");
 
@@ -158,12 +131,14 @@
         var indexName = $("#bookIndex").val();
         if(!indexName){
             $("#LabErr").html("章节名不能为空！");
+            lock = false;
             return;
         }
 
         var content = $("#bookContent").val();
-        if(!content && content.length<=13){
+        if(!content){
             $("#LabErr").html("章节内容不能为空！");
+            lock = false;
             return;
         }
 
@@ -182,11 +157,13 @@
 
 
                 } else {
+                    lock = false;
                     $("#LabErr").html(data.msg);
                 }
 
             },
             error: function () {
+                lock = false;
                 layer.alert('网络异常');
             }
         })

novel-front/src/main/resources/templates/book/book_ranking.html

@@ -22,7 +22,7 @@
 
                     <div class="book_tit">
                         <div class="fl">
-                            <h3 class="font26 mt5 mb5">点击榜</h3>
+                            <h3 class="font26 mt5 mb5" id="rankName">点击榜</h3>
                         </div>
                         <a class="fr"></a>
                     </div>
@@ -123,6 +123,25 @@
                 layer.alert('网络异常');
             }
         })
+        switch (rankType) {
+            case 0 : {
+                $("#rankName").html("点击榜");
+                break;
+            }
+            case 1 : {
+                $("#rankName").html("新书榜");
+                break;
+            }
+            case 2 : {
+                $("#rankName").html("更新榜");
+                break;
+            }
+            case 3 : {
+                $("#rankName").html("评论榜");
+                break;
+            }
+
+        }
 
     }
 </script>

novel-front/src/main/resources/templates/mobile/book/book_ranking.html

@@ -159,7 +159,7 @@
 
         if(sortBy != false){
             searchData.sort = sortBy;
-        }else{
+        }else if(!keyword){
             sortBy = "last_index_update_time";
         }
 
@@ -179,9 +179,13 @@
                     for (var i = 0; i < bookList.length; i++) {
                         var book = bookList[i];
 
-                        var end = book.bookDesc.indexOf("<");
+                        /*var end = book.bookDesc.indexOf("<");
                         if(end != -1) {
                             book.bookDesc = book.bookDesc.substring(0,end);
+                        }*/
+
+                        if(book.bookDesc){
+                            book.bookDesc = book.bookDesc.replace(/<[^>]+>/g,"").replace(/\s+/g,"").replace(/&nbsp;/g,"");
                         }
 
                         bookListHtml += ("<div class=\"layui-row\" style=\"margin-bottom:10px;padding:10px;background: #f2f2f2\">\n" +
@@ -202,7 +206,7 @@
                             "            </a>\n" +
                             "            <div style=\"margin-top: 5px;color: #4c6978;\">类别："+book.catName+"</div>\n" +
                             "            <div style=\"margin-top: 5px;color: #4c6978;\">状态："+(book.bookStatus==0?'连载':'完结')+"</div>\n" +
-                            "            <div style=\"margin-top: 5px;color: #4c6978;\">更新：<i>"+book.lastIndexUpdateTime+"</i>\n" +
+                            "            <div style=\"margin-top: 5px;color: #4c6978;\">更新：<i>"+book.lastIndexUpdateTime.substr(0,11)+"</i>\n" +
                             "            </div>\n" +
                             "            <div style=\"margin-top: 5px;color: #4c6978;\">简介："+(book.bookDesc?(book.bookDesc.length>20?(book.bookDesc.substr(0,20)+"..."):book.bookDesc):book.bookDesc)+"</div>\n" +
                             "\n" +

novel-front/src/main/resources/templates/mobile/index.html

@@ -58,13 +58,13 @@
 <ul  class="layui-nav app" lay-filter="" style="display:none;padding:0 10px;text-align: center">
     <li class="layui-nav-item"><a>分类</a>
         <dl class="layui-nav-child">
-            <dd><a href="/book/book_ranking.html?catId=1">玄幻小说</a></dd>
-            <dd><a href="/book/book_ranking.html?catId=2">修真小说</a></dd>
-            <dd><a href="/book/book_ranking.html?catId=3">都市小说</a></dd>
-            <dd><a href="/book/book_ranking.html?catId=4">历史小说</a></dd>
-            <dd><a href="/book/book_ranking.html?catId=6">网游小说</a></dd>
-            <dd><a href="/book/book_ranking.html?catId=5">科幻小说</a></dd>
-            <dd><a href="/book/book_ranking.html?catId=7">女频小说</a></dd>
+            <dd><a href="/book/book_ranking.html?catId=1&sort=last_index_update_time">玄幻小说</a></dd>
+            <dd><a href="/book/book_ranking.html?catId=2&sort=last_index_update_time">修真小说</a></dd>
+            <dd><a href="/book/book_ranking.html?catId=3&sort=last_index_update_time">都市小说</a></dd>
+            <dd><a href="/book/book_ranking.html?catId=4&sort=last_index_update_time">历史小说</a></dd>
+            <dd><a href="/book/book_ranking.html?catId=6&sort=last_index_update_time">网游小说</a></dd>
+            <dd><a href="/book/book_ranking.html?catId=5&sort=last_index_update_time">科幻小说</a></dd>
+            <dd><a href="/book/book_ranking.html?catId=7&sort=last_index_update_time">女频小说</a></dd>
         </dl>
     </li>
     <li class="layui-nav-item"><a>全本</a>
@@ -95,13 +95,13 @@
 
 </ul>
 <ul class="layui-nav pc" lay-filter="" style="padding:0 36px;text-align: center">
-    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=1">玄幻小说</a></li>
-    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=2">修真小说</a></li>
-    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=3">都市小说</a></li>
-    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=4">历史小说</a></li>
-    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=6">网游小说</a></li>
-    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=5">科幻小说</a></li>
-    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=7">女频小说</a></li>
+    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=1&sort=last_index_update_time">玄幻小说</a></li>
+    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=2&sort=last_index_update_time">修真小说</a></li>
+    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=3&sort=last_index_update_time">都市小说</a></li>
+    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=4&sort=last_index_update_time">历史小说</a></li>
+    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=6&sort=last_index_update_time">网游小说</a></li>
+    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=5&sort=last_index_update_time">科幻小说</a></li>
+    <li class="layui-nav-item"><a href="/book/book_ranking.html?catId=7&sort=last_index_update_time">女频小说</a></li>
     <li class="layui-nav-item"><a>完本小说</a>
         <dl class="layui-nav-child"> <!-- 二级菜单 -->
             <dd><a href="/book/book_ranking.html?bookStatus=1">全部小说</a></dd>
@@ -276,6 +276,10 @@
                 for (var i = 0; i < 6; i++) {
                     var hotRecBook = hotRecBooks[i];
 
+                    if(hotRecBook.bookDesc){
+                        hotRecBook.bookDesc = hotRecBook.bookDesc.replace(/<[^>]+>/g,"").replace(/\s+/g,"");
+                    }
+
                     hotRecBooksHtml += ("<div style=\"margin-bottom: 5px\" class=\"layui-col-xs12 layui-col-sm6 layui-col-md4 layui-col-lg4\">\n" +
                         "                        <a href=\"/book/"+hotRecBook.bookId+".html\">\n" +
                         "                            <div class=\"layui-col-xs5 layui-col-sm4 layui-col-md4 layui-col-lg4\" >\n" +
@@ -323,9 +327,8 @@
                 for (var i = 0; i < 10; i++) {
 
                     var updateRankBook = updateRankBooks[i];
-                    var end = updateRankBook.bookDesc.indexOf("<");
-                    if(end != -1) {
-                        updateRankBook.bookDesc = updateRankBook.bookDesc.substring(0,end);
+                    if(updateRankBook.bookDesc){
+                        updateRankBook.bookDesc = updateRankBook.bookDesc.replace(/<[^>]+>/g,"").replace(/\s+/g,"");
                     }
 
                         updateRankBookHtml += ("<div style=\"padding-bottom: 30px\"\n" +

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.java2nb</groupId>
     <artifactId>novel</artifactId>
-    <version>2.0.2</version>
+    <version>2.6.4</version>
     <modules>
         <module>novel-common</module>
         <module>novel-front</module>
@@ -36,6 +36,15 @@
         <orderbyhelper.version>1.0.2</orderbyhelper.version>
         <commons-lang3.version>3.4</commons-lang3.version>
         <jjwt.version>0.9.0</jjwt.version>
+        <elasticsearch.version>6.2.2</elasticsearch.version>
+        <jest.version>6.3.1</jest.version>
+        <redis.version>1.4.1.RELEASE</redis.version>
+        <redisson.version>3.12.5</redisson.version>
+        <sharding.jdbc.version>3.0.0</sharding.jdbc.version>
+        <aliyun-sdk-oss.version>2.4.0</aliyun-sdk-oss.version>
+        <commons-fileupload.version>1.3.1</commons-fileupload.version>
+        <fastdfs-client.version>1.26.1-RELEASE</fastdfs-client.version>
+        <alipay-sdk-java.version>4.9.153.ALL</alipay-sdk-java.version>
     </properties>
 
     <dependencyManagement>

sql/20200608.sql

@@ -0,0 +1,2 @@
+INSERT INTO `crawl_source` (`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES
+(6, '新笔趣阁', '{\n	\"bookListUrl\": \"http://www.xbiquge.la/fenlei/{catId}_{page}.html\",\n	\"catIdRule\": {\n		\"catId1\": \"1\",\n		\"catId2\": \"2\",\n		\"catId3\": \"3\",\n		\"catId4\": \"4\",\n		\"catId5\": \"6\",\n		\"catId6\": \"5\"\n	},\n	\"bookIdPatten\": \"<a\\\\s+href=\\\"http://www.xbiquge.la/(\\\\d+/\\\\d+)/\\\"\\\\s+target=\\\"_blank\\\">\",\n	\"pagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">(\\\\d+)/\\\\d+</em>\",\n	\"totalPagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">\\\\d+/(\\\\d+)</em>\",\n	\"bookDetailUrl\": \"http://www.xbiquge.la/{bookId}/\",\n	\"bookNamePatten\": \"<h1>([^/]+)</h1>\",\n	\"authorNamePatten\": \"者：([^/]+)</p>\",\n	\"picUrlPatten\": \"src=\\\"(http://www.xbiquge.la/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n	\"bookStatusRule\": {},\n	\"descStart\": \"<div id=\\\"intro\\\">\",\n	\"descEnd\": \"</div>\",\n	\"upadateTimePatten\": \"<p>最后更新：(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)</p>\",\n	\"upadateTimeFormatPatten\": \"yyyy-MM-dd HH:mm:ss\",\n	\"bookIndexUrl\": \"http://www.xbiquge.la/{bookId}/\",\n	\"indexIdPatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/(\\\\d+)\\\\.html\'\\\\s+>[^/]+</a>\",\n	\"indexNamePatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/\\\\d+\\\\.html\'\\\\s+>([^/]+)</a>\",\n	\"bookContentUrl\": \"http://www.xbiquge.la/{bookId}/{indexId}.html\",\n	\"contentStart\": \"<div id=\\\"content\\\">\",\n	\"contentEnd\": \"<p>\"\n}', 0, '2020-05-23 22:46:58', '2020-05-23 22:46:58');

sql/20200615.sql

@@ -0,0 +1,40 @@
+/*
+Navicat MySQL Data Transfer
+
+Source Server         : localhost
+Source Server Version : 50725
+Source Host           : localhost:3306
+Source Database       : novel_plus
+
+Target Server Type    : MYSQL
+Target Server Version : 50725
+File Encoding         : 65001
+
+Date: 2020-06-15 15:06:55
+*/
+
+SET FOREIGN_KEY_CHECKS=0;
+
+-- ----------------------------
+-- Table structure for crawl_single_task
+-- ----------------------------
+DROP TABLE IF EXISTS `crawl_single_task`;
+CREATE TABLE `crawl_single_task` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `source_id` int(11) DEFAULT NULL COMMENT '爬虫源ID',
+  `source_name` varchar(50) DEFAULT NULL COMMENT '爬虫源名',
+  `source_book_id` varchar(255) DEFAULT NULL COMMENT '源站小说ID',
+  `cat_id` int(11) DEFAULT NULL COMMENT '分类ID',
+  `book_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说名',
+  `author_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说作者名',
+  `task_status` tinyint(1) DEFAULT '2' COMMENT '任务状态，0：失败，1：成功，2；未执行',
+  `exc_count` tinyint(2) DEFAULT '0' COMMENT '已经执行次数，最多执行5次',
+  `create_time` datetime DEFAULT NULL COMMENT '创建时间',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='抓取单本小说任务表';
+
+-- ----------------------------
+-- Records of crawl_single_task
+-- ----------------------------
+INSERT INTO `crawl_single_task` VALUES ('6', '2', '百书斋', '1', '1', '1', '1', '0', '5', '2020-06-15 14:36:07');
+INSERT INTO `crawl_single_task` VALUES ('7', '5', '笔趣阁', '108_108291', '1', '衍天志之不朽仙', '白衣少年丶', '1', '1', '2020-06-15 14:46:08');

sql/novel_plus.sql

@@ -1691,4 +1691,148 @@ INSERT INTO `user_read_history` VALUES ('113', '1255060328322027520', '125467697
 INSERT INTO `user_read_history` VALUES ('117', '1255060328322027520', '1254946661743603712', '1254946914001629184', '2020-04-30 19:37:09', '2020-04-30 19:37:09');
 INSERT INTO `user_read_history` VALUES ('118', '1255060328322027520', '1254957312633352192', '3335449', '2020-04-30 19:37:36', '2020-04-30 19:37:36');
 
-UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n	\"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n	\"catIdRule\": {\n		\"catId1\": \"1\",\n		\"catId2\": \"2\",\n		\"catId3\": \"3\",\n		\"catId4\": \"4\",\n		\"catId5\": \"7\",\n		\"catId6\": \"6\",\n		\"catId7\": \"8\"\n	},\n	\"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n	\"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n	\"totalPagePatten\": \"第\\\\d+/(\\\\d+)页\",\n	\"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n	\"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n	\"authorNamePatten\": \"<p>作者：([^/]+)</p>\",\n	\"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n	\"statusPatten\": \"<p>状态：([^/]+)</p>\",\n	\"bookStatusRule\": {\n		\"连载中\": 0,\n		\"完本\": 1\n	},\n	\"descStart\": \"<div class=\\\"intro_info\\\">\",\n	\"descEnd\": \"最新章节推荐地址\",\n	\"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n	\"bookIndexStart\": \"<dt>《\",\n	\"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n	\"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n	\"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n	\"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n	\"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;
+
+
+DROP TABLE IF EXISTS `book_content0`;
+CREATE TABLE `book_content0` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1155 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content1
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content1`;
+CREATE TABLE `book_content1` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=406 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content2
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content2`;
+CREATE TABLE `book_content2` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1222 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content3
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content3`;
+CREATE TABLE `book_content3` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=410 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content4
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content4`;
+CREATE TABLE `book_content4` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1188 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content5
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content5`;
+CREATE TABLE `book_content5` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=416 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content6
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content6`;
+CREATE TABLE `book_content6` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1180 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content7
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content7`;
+CREATE TABLE `book_content7` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=404 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content8
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content8`;
+CREATE TABLE `book_content8` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1134 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content9
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content9`;
+CREATE TABLE `book_content9` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+
+DROP TABLE IF EXISTS `crawl_single_task`;
+CREATE TABLE `crawl_single_task` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `source_id` int(11) DEFAULT NULL COMMENT '爬虫源ID',
+  `source_name` varchar(50) DEFAULT NULL COMMENT '爬虫源名',
+  `source_book_id` varchar(255) DEFAULT NULL COMMENT '源站小说ID',
+  `cat_id` int(11) DEFAULT NULL COMMENT '分类ID',
+  `book_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说名',
+  `author_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说作者名',
+  `task_status` tinyint(1) DEFAULT '2' COMMENT '任务状态，0：失败，1：成功，2；未执行',
+  `exc_count` tinyint(2) DEFAULT '0' COMMENT '已经执行次数，最多执行5次',
+  `create_time` datetime DEFAULT NULL COMMENT '创建时间',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='抓取单本小说任务表';
+
+-- ----------------------------
+-- Records of crawl_single_task
+-- ----------------------------
+INSERT INTO `crawl_single_task` VALUES ('6', '2', '百书斋', '1', '1', '1', '1', '0', '5', '2020-06-15 14:36:07');
+INSERT INTO `crawl_single_task` VALUES ('7', '5', '笔趣阁', '108_108291', '1', '衍天志之不朽仙', '白衣少年丶', '1', '1', '2020-06-15 14:46:08');
+UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n	\"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n	\"catIdRule\": {\n		\"catId1\": \"1\",\n		\"catId2\": \"2\",\n		\"catId3\": \"3\",\n		\"catId4\": \"4\",\n		\"catId5\": \"7\",\n		\"catId6\": \"6\",\n		\"catId7\": \"8\"\n	},\n	\"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n	\"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n	\"totalPagePatten\": \"第\\\\d+/(\\\\d+)页\",\n	\"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n	\"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n	\"authorNamePatten\": \"<p>作者：([^/]+)</p>\",\n	\"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n	\"statusPatten\": \"<p>状态：([^/]+)</p>\",\n	\"bookStatusRule\": {\n		\"连载中\": 0,\n		\"完本\": 1\n	},\n	\"descStart\": \"<div class=\\\"intro_info\\\">\",\n	\"descEnd\": \"最新章节推荐地址\",\n	\"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n	\"bookIndexStart\": \"<dt>《\",\n	\"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n	\"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n	\"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n	\"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n	\"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;
+
+INSERT INTO `friend_link` ( `link_name`, `link_url`, `sort`, `is_open`, `create_user_id`, `create_time`, `update_user_id`, `update_time`) VALUES
+('小羊影视', 'http://video.java2nb.com/', 11, 1, NULL, NULL, NULL, NULL),
+('官方论坛', 'http://bbs.java2nb.com', 21, 1, NULL, NULL, NULL, NULL);