增加新源，过滤简介中的特殊字符，提高爬虫兼容性

README.md

@@ -1,7 +1,15 @@
-[![index](./assets/热门云产品1040.100.jpg)](https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console)
+[![index]( https://i.loli.net/2020/06/01/qF1uBMtswTIPJKx.jpg )]( https://www.eve.bz/cart.php?gid=11 )
 
 # 小说精品屋-plus
 
+#### 新项目：小说精品屋-微服务版
+
+基于小说精品屋-plus构建的Spring Cloud 微服务小说门户平台，可用于学习和商用。
+
+Gitee仓库地址： https://gitee.com/xiongxyang/novel-cloud
+
+GitHub仓库地址： https://github.com/201206030/novel-cloud
+
 #### 演示地址
 
 [点击前往](http://www.java2nb.com)（前台门户）
@@ -40,7 +48,31 @@ novel-plus -- 父工程
 ```
 
 #### 技术选型
-Springboot+Mybatis+Mysql+ElasticSearch+Ehcache+Thymeleaf+Layui
+
+| 技术                 | 说明                                                         
+| -------------------- | ---------------------------
+| SpringBoot           | Spring应用快速开发脚手架     
+| MyBatis              | 持久层ORM框架 
+| MyBatis Dynamic SQL  | Mybatis动态sql
+| PageHelper           | MyBatis分页插件
+| MyBatisGenerator     | 持久层代码生成插件
+| Sharding-Jdbc        | 代码层分库分表中间件
+| JJWT                 | JWT登录支持  
+| SpringSecurity       | 安全框架                           
+| Shiro                | 安全框架  
+| Ehcache              | Java进程内缓存框架(默认缓存)  
+| Redis                | 分布式缓存(缓存替换方案，默认关闭，一行配置开启)                               
+| ElasticSearch        | 搜索引擎(搜索增强方案，默认关闭，一行配置开启)                      
+| RabbitMq             | 消息队列(流量削峰，默认关闭，一行配置开启)  
+| OSS                  | 阿里云对象存储服务(图片存储方式之一，一行配置即可切换) 
+| FastDfs              |开源轻量级分布式文件系统(图片存储方式之一，一行配置即可切换)                      
+| Redisson             | 实现分布式锁                                       
+| Lombok               | 简化对象封装工具                                                                               
+| Docker               | 应用容器引擎   
+| Mysql                | 数据库服务   
+| Thymeleaf            | 模板引擎     
+| Layui                | 前端UI                    
+                 
 
 #### PC站截图
 
@@ -54,11 +86,9 @@ Springboot+Mybatis+Mysql+ElasticSearch+Ehcache+Thymeleaf+Layui
 
 3. 搜索页
 
-   
+![img](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/QQ20200520-215756.png)
 
-   ![img](./assets/QQ20200520-215756.png)
-
-   
+![img](https://oscimg.oschina.net/oscnet/up-ed5f689557718924acac76bc3ebca36afcb.png)
 
 4. 排行榜
 
@@ -94,35 +124,27 @@ Springboot+Mybatis+Mysql+ElasticSearch+Ehcache+Thymeleaf+Layui
 
 ![img](https://oscimg.oschina.net/oscnet/up-f849960f4c1303fea77d26e64fc505a7180.png)
 
-
-
 #### 手机站截图
 
 1. 首页
 
-   ![index](./assets/QQ%E5%9B%BE%E7%89%8720191018162208.jpg)
-
-   
+   ![index](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/QQ%E5%9B%BE%E7%89%8720191018162208.jpg)
 
 2. 小说详情页
 
-   ![微信图片_20190904181558](./assets/%E5%BE%AE%E4%BF%A1%E5%9B%BE%E7%89%87_20190904181558.png)
-
-   
+   ![微信图片_20190904181558](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/%E5%BE%AE%E4%BF%A1%E5%9B%BE%E7%89%87_20190904181558.png)
 
 3. 目录页
 
-   ![QQ图片20191018161901](./assets/QQ%E5%9B%BE%E7%89%8720191108022250.png)
+   ![QQ图片20191018161901](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/QQ%E5%9B%BE%E7%89%8720191108022250.png)
 
 4. 小说阅读页
 
-   ![QQ图片20191018161901](./assets/QQ%E5%9B%BE%E7%89%8720191018161901.png)
-
-   
+   ![QQ图片20191018161901](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.0/assets/QQ%E5%9B%BE%E7%89%8720191018161901.png)
 
 #### 爬虫管理系统截图
 
-![QQ图片20191018161901](./assets/crawl_index.png)
+![img](https://gitee.com/xiongxyang/novel-plus/raw/release_v2.1.2/assets/crawl_index.png)
 
 #### 后台管理系统截图
 

novel-admin/Dockerfile

@@ -0,0 +1,6 @@
+FROM java:8
+ADD novel-admin-1.0.0.jar /root
+ENV dburl=""
+ENV username=""
+ENV	password=""
+ENTRYPOINT ["sh","-c","java -Dspring.datasource.url=${dburl} -Dspring.datasource.username=${username} -Dspring.datasource.password=${password} -jar /root/novel-admin-1.0.0.jar"]

novel-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.1.1</version>
+        <version>2.5.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -31,11 +31,20 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-cache</artifactId>
         </dependency>
+
+        <!--ehcache-->
         <dependency>
             <groupId>net.sf.ehcache</groupId>
             <artifactId>ehcache</artifactId>
         </dependency>
 
+        <!--集成redis-->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-redis</artifactId>
+            <version>${redis.version}</version>
+        </dependency>
+
         <!--mysql驱动-->
         <dependency>
             <groupId>mysql</groupId>
@@ -48,6 +57,22 @@
             <version>${mybatis.version}</version>
         </dependency>
 
+        <!-- 分库分表-->
+        <!-- sharding jdbc依赖 -->
+
+        <dependency>
+            <groupId>io.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-spring-boot-starter</artifactId>
+            <version>${sharding.jdbc.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-spring-namespace</artifactId>
+            <version>${sharding.jdbc.version}</version>
+        </dependency>
+
+
         <!--分页助手启动器-->
         <dependency>
             <groupId>com.github.pagehelper</groupId>

novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java

@@ -50,4 +50,19 @@ public interface CacheKey {
      * 上一次搜索引擎更新的时间
      * */
     String ES_LAST_UPDATE_TIME = "esLastUpdateTime";
+
+    /**
+     * 搜索引擎转换锁
+     * */
+    String ES_TRANS_LOCK = "esTransLock";
+
+    /**
+     * 上一次搜索引擎是否更新过小说点击量
+     * */
+    String ES_IS_UPDATE_VISIT = "esIsUpdateVisit";
+
+    /**
+     * 累积的小说点击量
+     * */
+    String BOOK_ADD_VISIT_COUNT = "bookAddVisitCount";
 }

novel-common/src/main/java/com/java2nb/novel/core/cache/CacheService.java

@@ -51,9 +51,5 @@ public interface CacheService {
 	 * */
 	void expire(String key, long timeout);
 
-	/**
-	 * 刷新缓存
-	 * */
-	void refresh(String key);
 
 }

novel-common/src/main/java/com/java2nb/novel/core/cache/impl/EhCacheServiceImpl.java

@@ -1,24 +1,25 @@
 package com.java2nb.novel.core.cache.impl;
 
 import com.java2nb.novel.core.cache.CacheService;
+import lombok.RequiredArgsConstructor;
 import net.sf.ehcache.Cache;
 import net.sf.ehcache.CacheManager;
 import net.sf.ehcache.Element;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Service;
 
 /**
  * @author xxy
  */
+@ConditionalOnProperty(prefix = "cache", name = "type", havingValue = "ehcache")
+@RequiredArgsConstructor
 @Service
 public class EhCacheServiceImpl implements CacheService {
 	
-	@Autowired
-	private CacheManager cacheManager ;
+	private final CacheManager cacheManager ;
  
-	private static final String CACHE_NAME = "utilCache";
- 
-	
+
 	/**
 	 * 获得一个Cache，没有则创建一个。
 	 * @return
@@ -30,14 +31,6 @@ public class EhCacheServiceImpl implements CacheService {
 	}
 	
  
-	public  CacheManager getCacheManager() {
-		return cacheManager;
-	}
-	
-	
-
-
-
 	@Override
 	public String get(String key) {
 		Element element = getCache().get(key);
@@ -125,20 +118,6 @@ public class EhCacheServiceImpl implements CacheService {
 	}
 
 
-	@Override
-	public void refresh(String key) {
-		Element element = getCache().get(key);
-		if (element != null) {
-			Object value = element.getValue();
-			int timeToLive = element.getTimeToLive();
-			element = new Element(key, value);
-			element.setTimeToLive(timeToLive);
-			Cache cache = getCache();
-			cache.put(element);
-		}
-		
-	}
-
 
 
 }

novel-common/src/main/java/com/java2nb/novel/core/cache/impl/RedisServiceImpl.java

@@ -0,0 +1,73 @@
+package com.java2nb.novel.core.cache.impl;
+
+import com.java2nb.novel.core.cache.CacheService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Service;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author xxy
+ */
+@ConditionalOnProperty(prefix = "cache", name = "type", havingValue = "redis")
+@RequiredArgsConstructor
+@Service
+public class RedisServiceImpl implements CacheService {
+
+	private final StringRedisTemplate stringRedisTemplate;
+
+	private final RedisTemplate<Object,  Object>  redisTemplate;
+
+
+	@Override
+	public String get(String key) {
+		return stringRedisTemplate.opsForValue().get(key);
+	}
+
+	@Override
+	public void set(String key, String value) {
+		stringRedisTemplate.opsForValue().set(key,value);
+	}
+
+	@Override
+	public void set(String key, String value, long timeout) {
+		stringRedisTemplate.opsForValue().set(key,value,timeout, TimeUnit.SECONDS);
+
+	}
+
+	@Override
+	public Object getObject(String key) {
+		return redisTemplate.opsForValue().get(key);
+	}
+
+	@Override
+	public void setObject(String key, Object value) {
+		redisTemplate.opsForValue().set(key,value);
+	}
+
+	@Override
+	public void setObject(String key, Object value, long timeout) {
+		redisTemplate.opsForValue().set(key,value,timeout, TimeUnit.SECONDS);
+	}
+
+	@Override
+	public void del(String key) {
+		redisTemplate.delete(key);
+		stringRedisTemplate.delete(key);
+	}
+
+	@Override
+	public boolean contains(String key) {
+		return redisTemplate.hasKey(key) || stringRedisTemplate.hasKey(key);
+	}
+
+	@Override
+	public void expire(String key, long timeout) {
+		redisTemplate.expire(key,timeout, TimeUnit.SECONDS);
+		stringRedisTemplate.expire(key,timeout, TimeUnit.SECONDS);
+	}
+
+}

novel-common/src/main/java/com/java2nb/novel/core/enums/ResponseStatus.java

@@ -58,13 +58,16 @@ public enum ResponseStatus {
     , BOOKNAME_EXISTS(4003,"已发布过同名小说！")
 
     ,
+    /**
+     * 搜索引擎相关错误
+     * */
+    ES_SEARCH_FAIL(9001,"密码错误！"),
 
 
     /**
      * 其他通用错误
      * */
     PASSWORD_ERROR(88001,"密码错误！");
-
     private int code;
     private String msg;
 

novel-common/src/main/java/com/java2nb/novel/core/utils/FileUtil.java

@@ -10,6 +10,7 @@ import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
 
+import javax.imageio.ImageIO;
 import java.io.*;
 import java.util.Date;
 import java.util.Objects;
@@ -48,6 +49,11 @@ public class FileUtil {
                 out.write(b, 0, n);
             }
 
+            out.flush();
+            if( ImageIO.read(picFile) == null){
+                picSrc = "/images/default.gif";
+            }
+
         }catch (Exception e){
             log.error(e.getMessage(),e);
 

novel-common/src/main/java/com/java2nb/novel/mapper/BookContentDynamicSqlSupport.java

@@ -24,7 +24,7 @@ public final class BookContentDynamicSqlSupport {
 
         public final SqlColumn<Long> indexId = column("index_id", JDBCType.BIGINT);
 
-        public final SqlColumn<String> content = column("content", JDBCType.LONGVARCHAR);
+        public final SqlColumn<String> content = column("content", JDBCType.VARCHAR);
 
         public BookContent() {
             super("book_content");

novel-common/src/main/java/com/java2nb/novel/mapper/BookContentMapper.java

@@ -63,7 +63,7 @@ public interface BookContentMapper {
     @Results(id="BookContentResult", value = {
         @Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
         @Result(column="index_id", property="indexId", jdbcType=JdbcType.BIGINT),
-        @Result(column="content", property="content", jdbcType=JdbcType.LONGVARCHAR)
+        @Result(column="content", property="content", jdbcType=JdbcType.VARCHAR)
     })
     List<BookContent> selectMany(SelectStatementProvider selectStatement);
 

novel-common/src/main/resources/application-common-dev.yml

@@ -1,17 +1,79 @@
 spring:
   profiles:
     include: [common]
+  main:
+    allow-bean-definition-overriding: true
+  #Redis服务器IP
+  redis:
+    host: 127.0.0.1
+    #Redis服务器连接端口
+    port: 6379
+    #Redis服务器连接密码
+    password: test
+    jedis:
+      pool:
+        #连接池最大连接数（使用负值表示没有限制）
+        max-active: 8
+        #连接池最大阻塞等待时间（使用负值表示没有限制）
+        max-wait: 1
+        #连接池最大阻塞等待时间（使用负值表示没有限制）
+        max-idle: 8
+        #连接池中的最小空闲连接
+        min-idle: 0
+    #连接超时时间（毫秒）
+    timeout: 30000
   datasource:
     url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
     username: root
     password: test123456
     driver-class-name: com.mysql.cj.jdbc.Driver
 
+####使用shardingJdbc时，
+####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误
+##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR
+
+sharding:
+  jdbc:
+    datasource:
+      names: ds0 #,ds1
+      ds0:
+        type: com.zaxxer.hikari.HikariDataSource
+        driver-class-name: com.mysql.cj.jdbc.Driver
+        jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
+        username: root
+        password: test123456
+#      ds1:
+#        type: com.alibaba.druid.pool.DruidDataSource
+#        driver-class-name: com.mysql.jdbc.Driver
+#        url: jdbc:mysql://localhost:3306/novel_plus2
+#        username: root
+#        password: test123456
+    config:
+      sharding:
+        props:
+          sql.show: true
+        tables:
+          book_content:  #book_content表
+            key-generator-column-name: id  #主键
+            actual-data-nodes: ds${0}.book_content${0..9}  #数据节点
+#            database-strategy:  #分库策略
+#              inline:
+#                sharding-column: book_id
+#                algorithm-expression: ds${book_id % 10}
+            table-strategy:  #分表策略
+              inline:
+                shardingColumn: index_id
+                algorithm-expression: book_content${index_id % 10}
+
+
+
+
 
 
 pic:
   save:
-    type: 2 #图片保存方式, 1不保存，使用网络图片 ，2本地保存
+    type: 2 #图片保存方式, 1不保存，使用爬取的网络图片 ，2保存在自己的存储介质
+    storage: local #存储介质，local：本地，OSS：阿里云对象存储，fastDfs：分布式文件系统
     path: /var/pic  #图片保存路径
 
 

novel-common/src/main/resources/application-common.yml

@@ -11,6 +11,9 @@ spring:
     generator:
       write-numbers-as-strings: true
 
+#缓存类型，ehcache(默认)、redis
+cache:
+  type: ehcache
 
 mybatis:
   configuration:

novel-crawl/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.1.1</version>
+        <version>2.5.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -25,6 +25,8 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-security</artifactId>
         </dependency>
+
+
     </dependencies>
 
     <build>

novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java

@@ -30,6 +30,8 @@ import static java.util.regex.Pattern.compile;
 @Slf4j
 public class CrawlParser {
 
+    private static IdWorker idWorker = new IdWorker();
+
     public static final Integer BOOK_INDEX_LIST_KEY = 1;
 
     public static final Integer BOOK_CONTENT_LIST_KEY = 2;
@@ -94,6 +96,12 @@ public class CrawlParser {
 
                     String desc = bookDetailHtml.substring(bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length());
                     desc = desc.substring(0, desc.indexOf(ruleBean.getDescEnd()));
+                    //过滤掉简介中的特殊标签
+                    desc = desc.replaceAll("<a[^<]+</a>","")
+                            .replaceAll("<font[^<]+</font>","")
+                            .replaceAll("<p>\\s*</p>","")
+                            .replaceAll("<p>","")
+                            .replaceAll("</p>","<br/>");
                     //设置书籍简介
                     book.setBookDesc(desc);
                     if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {
@@ -173,6 +181,7 @@ public class CrawlParser {
             String lastIndexName = null;
 
             while (isFindIndex) {
+
                 BookIndex hasIndex = hasIndexs.get(indexNum);
                 String indexName = indexNameMatch.group(1);
 
@@ -199,7 +208,7 @@ public class CrawlParser {
                         if(hasIndexs.size() == 0){
                             //新书入库
                             //设置目录和章节内容
-                            Long indexId = new IdWorker().nextId();
+                            Long indexId = idWorker.nextId();
                             lastIndexId = indexId;
                             lastIndexName = indexName;
                             bookIndex.setId(indexId);

novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java

@@ -61,7 +61,7 @@ public class StarterListener implements ServletContextListener {
                                 Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
                                 //这里只做老书更新
                                 book.setId(needUpdateBook.getId());
-                                if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
+                                if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
                                     //本地图片则不更新
                                     book.setPicUrl(null);
                                 }

novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java

@@ -167,6 +167,11 @@ public class CrawlServiceImpl implements CrawlService {
                         boolean isFindBookId = bookIdMatcher.find();
                         while (isFindBookId) {
                             try {
+                                if(Thread.currentThread().isInterrupted()){
+                                    return;
+                                }
+
+
                                 String bookId = bookIdMatcher.group(1);
                                 Book book = CrawlParser.parseBook(ruleBean, bookId);
                                 //这里只做新书入库，查询是否存在这本书

novel-front/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>novel</artifactId>
         <groupId>com.java2nb</groupId>
-        <version>2.1.1</version>
+        <version>2.5.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -27,6 +27,11 @@
             <version>${jjwt.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-amqp</artifactId>
+        </dependency>
+
 
         <dependency>
             <groupId>io.searchbox</groupId>
@@ -34,16 +39,45 @@
             <version>${jest.version}</version>
         </dependency>
 
+        <!--aliyunOSS-->
+        <dependency>
+            <groupId>com.aliyun.oss</groupId>
+            <artifactId>aliyun-sdk-oss</artifactId>
+            <version>${aliyun-sdk-oss.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-fileupload</groupId>
+            <artifactId>commons-fileupload</artifactId>
+            <version>${commons-fileupload.version}</version>
+        </dependency>
+
+        <!--FastDfs-->
+        <dependency>
+            <groupId>com.github.tobato</groupId>
+            <artifactId>fastdfs-client</artifactId>
+            <version>${fastdfs-client.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
         </dependency>
 
 
+
+
+        <!--引入redisson分布式锁-->
+       <!-- <dependency>
+            <groupId>org.redisson</groupId>
+            <artifactId>redisson-spring-boot-starter</artifactId>
+            <version>${redisson.version}</version>
+        </dependency>-->
+
+
         <dependency>
             <groupId>com.alipay.sdk</groupId>
             <artifactId>alipay-sdk-java</artifactId>
-            <version>4.9.153.ALL</version>
+            <version>${alipay-sdk-java.version}</version>
         </dependency>
     </dependencies>
 

novel-front/src/main/java/com/java2nb/novel/FrontNovelApplication.java

@@ -1,10 +1,13 @@
 package com.java2nb.novel;
 
+import com.github.tobato.fastdfs.FdfsClientConfig;
+import io.shardingsphere.shardingjdbc.spring.boot.SpringBootConfiguration;
 import org.mybatis.spring.annotation.MapperScan;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.TaskScheduler;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
@@ -18,6 +21,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
 @EnableScheduling
 @EnableCaching
 @MapperScan(basePackages = {"com.java2nb.novel.mapper"})
+@Import(FdfsClientConfig.class)
 public class FrontNovelApplication {
 
     public static void main(String[] args) {

novel-front/src/main/java/com/java2nb/novel/controller/BookController.java

@@ -4,14 +4,14 @@ import com.github.pagehelper.PageInfo;
 import com.java2nb.novel.core.bean.ResultBean;
 import com.java2nb.novel.core.bean.UserDetails;
 import com.java2nb.novel.core.enums.ResponseStatus;
-import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.entity.BookComment;
-import com.java2nb.novel.entity.BookIndex;
 import com.java2nb.novel.search.BookSP;
 import com.java2nb.novel.service.BookService;
 import com.java2nb.novel.vo.BookVO;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -19,7 +19,6 @@ import org.springframework.web.bind.annotation.RestController;
 
 import javax.servlet.http.HttpServletRequest;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -33,6 +32,11 @@ public class BookController extends BaseController{
 
     private final BookService bookService;
 
+    private final RabbitTemplate rabbitTemplate;
+
+    @Value("${spring.rabbitmq.enable}")
+    private Integer enableMq;
+
 
     /**
      * 查询首页小说设置列表数据
@@ -105,7 +109,11 @@ public class BookController extends BaseController{
      * */
     @PostMapping("addVisitCount")
     public ResultBean addVisitCount(Long bookId){
-        bookService.addVisitCount(bookId);
+        if(enableMq == 1) {
+            rabbitTemplate.convertAndSend("ADD-BOOK-VISIT-EXCHANGE", null, bookId);
+        }else {
+            bookService.addVisitCount(bookId, 1);
+        }
         return ResultBean.ok();
     }
 

novel-front/src/main/java/com/java2nb/novel/core/config/FdfsConfiguration.java

@@ -0,0 +1,13 @@
+package com.java2nb.novel.core.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.EnableMBeanExport;
+import org.springframework.jmx.support.RegistrationPolicy;
+
+/**
+ * @author 11797
+ */
+@Configuration
+@EnableMBeanExport(registration= RegistrationPolicy.IGNORE_EXISTING)
+public class FdfsConfiguration {
+}

novel-front/src/main/java/com/java2nb/novel/core/config/OssProperties.java

@@ -0,0 +1,30 @@
+package com.java2nb.novel.core.config;
+
+import lombok.Data;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author 11797
+ */
+@Data
+@Component
+@ConfigurationProperties(prefix="novel.file")
+public class OssProperties{
+
+    private String endpoint;
+
+    private String keyId;
+
+    private String keySecret;
+
+    private String fileHost;
+
+    private String bucketName;
+
+    private String webUrl;
+
+
+}

novel-front/src/main/java/com/java2nb/novel/core/config/RabbitConfig.java

@@ -0,0 +1,59 @@
+package com.java2nb.novel.core.config;
+
+import org.springframework.amqp.core.*;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author 11797
+ */
+@Configuration
+@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "enable", havingValue = "1")
+public class RabbitConfig {
+
+
+    /**
+     * 更新数据库队列
+     */
+    @Bean
+    public Queue updateDbQueue() {
+        return new Queue("UPDATE-DB-QUEUE", true);
+    }
+
+    /**
+     * 更新数据库队列
+     */
+    @Bean
+    public Queue updateEsQueue() {
+        return new Queue("UPDATE-ES-QUEUE", true);
+    }
+
+
+    /**
+     * 增加点击量交换机
+     */
+    @Bean
+    public FanoutExchange addVisitExchange() {
+        return new FanoutExchange("ADD-BOOK-VISIT-EXCHANGE");
+    }
+
+    /**
+     * 更新搜索引擎队列绑定到增加点击量交换机中
+     */
+    @Bean
+    public Binding updateEsBinding() {
+
+        return BindingBuilder.bind(updateEsQueue()).to(addVisitExchange());
+    }
+
+    /**
+     * 更新数据库绑定到增加点击量交换机中
+     */
+    @Bean
+    public Binding updateDbBinding() {
+        return BindingBuilder.bind(updateDbQueue()).to(addVisitExchange());
+    }
+
+
+}

novel-front/src/main/java/com/java2nb/novel/core/listener/BookVisitAddListener.java

@@ -0,0 +1,107 @@
+package com.java2nb.novel.core.listener;
+
+import com.java2nb.novel.core.cache.CacheKey;
+import com.java2nb.novel.core.cache.CacheService;
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.service.BookService;
+import com.java2nb.novel.service.SearchService;
+import com.java2nb.novel.vo.EsBookVO;
+import com.rabbitmq.client.Channel;
+import io.searchbox.client.JestClient;
+import io.searchbox.core.Index;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.core.Message;
+import org.springframework.beans.BeanUtils;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+
+
+import java.text.SimpleDateFormat;
+
+
+/**
+ * @author 11797
+ */
+@Component
+@Slf4j
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "enable", havingValue = "1")
+public class BookVisitAddListener {
+
+    private final BookService bookService;
+
+    private final CacheService cacheService;
+
+    private final SearchService searchService;
+
+   // private final RedissonClient redissonClient;
+
+
+
+
+    /**
+     * 更新数据库
+     * 流量削峰，每本小说累积10个点击更新一次
+     */
+    @SneakyThrows
+    @RabbitListener(queues = {"UPDATE-DB-QUEUE"})
+    public void updateDb(Long bookId, Channel channel, Message message) {
+
+        log.debug("收到更新数据库消息：" + bookId);
+        Thread.sleep(1000 * 2);
+        //TODO 操作共享资源visitCount，集群环境下有线程安全问题，引入Redisson框架实现分布式锁
+        //RLock lock = redissonClient.getLock("visitCount");
+        //lock.lock();
+
+
+        //目前visitCount不重要，数据可丢失，暂不实现分布式锁
+        Integer visitCount = (Integer) cacheService.getObject(CacheKey.BOOK_ADD_VISIT_COUNT+bookId);
+        if(visitCount == null){
+            visitCount = 0 ;
+        }
+        cacheService.setObject(CacheKey.BOOK_ADD_VISIT_COUNT+bookId,++visitCount);
+        if(visitCount >= Constants.ADD_MAX_VISIT_COUNT) {
+            bookService.addVisitCount(bookId,visitCount);
+            cacheService.del(CacheKey.BOOK_ADD_VISIT_COUNT+bookId);
+        }
+
+        //TODO 操作共享资源visitCount，集群环境下有线程安全问题，引入Redisson框架实现分布式锁
+        //lock.unlock();
+
+
+
+    }
+
+    /**
+     * 更新搜索引擎
+     * 流量削峰，每本小说1个小时更新一次
+     */
+    @RabbitListener(queues = {"UPDATE-ES-QUEUE"})
+    public void updateEs(Long bookId, Channel channel, Message message) {
+
+        log.debug("收到更新搜索引擎消息：" + bookId);
+        if (cacheService.get(CacheKey.ES_IS_UPDATE_VISIT + bookId) == null) {
+            cacheService.set(CacheKey.ES_IS_UPDATE_VISIT + bookId, "1", 60 * 60);
+            try {
+                Thread.sleep(1000 * 5);
+                Book book = bookService.queryBookDetail(bookId);
+                searchService.importToEs(book);
+            }catch (Exception e){
+                cacheService.del(CacheKey.ES_IS_UPDATE_VISIT + bookId);
+                log.error("更新搜索引擎失败"+bookId);
+            }
+
+
+
+
+        }
+
+
+    }
+
+
+}

novel-front/src/main/java/com/java2nb/novel/core/schedule/BookToEsSchedule.java

@@ -5,6 +5,7 @@ import com.java2nb.novel.core.cache.CacheService;
 import com.java2nb.novel.core.utils.BeanUtil;
 import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.service.BookService;
+import com.java2nb.novel.service.SearchService;
 import com.java2nb.novel.vo.EsBookVO;
 import io.searchbox.client.JestClient;
 import io.searchbox.core.DocumentResult;
@@ -37,18 +38,20 @@ public class BookToEsSchedule {
 
     private final CacheService cacheService;
 
-    private final JestClient jestClient;
 
 
-    private boolean lock = false;
+    private final SearchService searchService;
+
 
     /**
-     * 2分钟导入一次
+     * 1分钟导入一次
      */
-    @Scheduled(fixedRate = 1000 * 60 * 2)
+    @Scheduled(fixedRate = 1000 * 60)
     public void saveToEs() {
-        if (!lock) {
-            lock = true;
+        //TODO 引入Redisson框架实现分布式锁
+        //可以重复更新，只是效率可能略有降低，所以暂不实现分布式锁
+        if (cacheService.get(CacheKey.ES_TRANS_LOCK) == null) {
+            cacheService.set(CacheKey.ES_TRANS_LOCK, "1", 60 * 20);
             try {
                 //查询需要更新的小说
                 Date lastDate = (Date) cacheService.getObject(CacheKey.ES_LAST_UPDATE_TIME);
@@ -56,37 +59,25 @@ public class BookToEsSchedule {
                     lastDate = new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-01");
                 }
 
-                long count ;
-                do {
 
-                    List<Book> books = bookService.queryBookByUpdateTimeByPage(lastDate,100);
-                    for(Book book : books) {
-                        //导入到ES
-                        EsBookVO esBookVO = new EsBookVO();
-                        BeanUtils.copyProperties(book,esBookVO,"lastIndexUpdateTime");
-                        esBookVO.setLastIndexUpdateTime(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(book.getLastIndexUpdateTime()));
-                        Index action = new Index.Builder(esBookVO).index("novel").type("book").id(book.getId().toString()).build();
+                List<Book> books = bookService.queryBookByUpdateTimeByPage(lastDate, 100);
+                for (Book book : books) {
+                    searchService.importToEs(book);
+                    lastDate = book.getUpdateTime();
+                    Thread.sleep(5000);
 
-                        jestClient.execute(action);
+                }
 
-                        lastDate = book.getUpdateTime();
-                        //1秒钟导入一本书，1分钟导入60本
-                        Thread.sleep(1000);
 
-                    }
-
-                    count = books.size();
-
-                }while (count == 100);
 
                 cacheService.setObject(CacheKey.ES_LAST_UPDATE_TIME, lastDate);
 
             } catch (Exception e) {
-                log.error(e.getMessage(),e);
+                log.error(e.getMessage(), e);
             }
+            cacheService.del(CacheKey.ES_TRANS_LOCK);
 
 
-            lock = false;
         }
 
 

novel-front/src/main/java/com/java2nb/novel/core/schedule/CrawlPicTransSchedule.java

@@ -1,5 +1,6 @@
 package com.java2nb.novel.core.schedule;
 
+import com.java2nb.novel.core.utils.Constants;
 import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.service.BookService;
 import lombok.RequiredArgsConstructor;
@@ -13,7 +14,7 @@ import org.springframework.stereotype.Service;
 import java.util.List;
 
 /**
- * 网络图片转存本地任务
+ * 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）任务
  *
  * @author Administrator
  */
@@ -21,7 +22,7 @@ import java.util.List;
 @Service
 @RequiredArgsConstructor
 @Slf4j
-public class Network2LocalPicSchedule {
+public class CrawlPicTransSchedule {
 
     private final BookService bookService;
 
@@ -41,7 +42,7 @@ public class Network2LocalPicSchedule {
         log.info("Network2LocalPicSchedule。。。。。。。。。。。。");
 
 
-        List<Book> networkPicBooks = bookService.queryNetworkPicBooks(100);
+        List<Book> networkPicBooks = bookService.queryNetworkPicBooks(Constants.LOCAL_PIC_PREFIX,100);
         for (Book book : networkPicBooks) {
             bookService.updateBookPicToLocal(book.getPicUrl(), book.getId());
             //3秒钟转化一张图片，10分钟转化200张

novel-front/src/main/java/com/java2nb/novel/core/utils/Constants.java

@@ -12,7 +12,7 @@ public class Constants {
     public static final String TEMPLATE_PATH_PREFIX_KEY = "templatePathPrefixKey";
 
     /**
-     * 本地图片保存前缀
+     * 保存图片到自己的存储介质路径前缀
      * */
     public static final String LOCAL_PIC_PREFIX = "/localPic/";
 
@@ -30,4 +30,9 @@ public class Constants {
      * 首页设置的小说数量
      * */
     public static final int INDEX_BOOK_SETTING_NUM = 32;
+
+    /**
+     * 累积的最大点击量
+     * */
+    public static final Integer ADD_MAX_VISIT_COUNT = 10;
 }

novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java

@@ -16,13 +16,13 @@ public interface FrontBookMapper extends BookMapper {
 
     List<BookVO> searchByPage(BookSP params);
 
-    void addVisitCount(@Param("bookId") Long bookId, @Param("date") Date date);
+    void addVisitCount(@Param("bookId") Long bookId, @Param("visitCount") Integer visitCount);
 
     List<Book> listRecBookByCatId(@Param("catId") Integer catId);
 
     void addCommentCount(@Param("bookId") Long bookId);
 
-    List<Book> queryNetworkPicBooks(@Param("limit") Integer limit);
+    List<Book> queryNetworkPicBooks(@Param("localPicPrefix") String localPicPrefix, @Param("limit") Integer limit);
 
     /**
      * 按评分随机查询小说集合

novel-front/src/main/java/com/java2nb/novel/service/BookService.java

@@ -112,8 +112,8 @@ public interface BookService {
     /**
      * 增加点击次数
      * @param bookId 书籍ID
-     * */
-    void addVisitCount(Long bookId);
+     * @param visitCount*/
+    void addVisitCount(Long bookId, Integer visitCount);
 
     /**
      * 查询章节数
@@ -180,15 +180,17 @@ public interface BookService {
 
     /**
      * 查询网络图片的小说
+     *
+     * @param localPicPrefix
      * @param limit 查询条数
      * @return 返回小说集合
      * */
-    List<Book> queryNetworkPicBooks(Integer limit);
+    List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit);
 
 
     /**
-     * 更新小说网络图片到本地
-     * @param picUrl 网络图片路径
+     * 更新爬取的小说网络图片到自己的存储介质（本地、OSS、fastDfs）
+     * @param picUrl 爬取的网络图片路径
      * @param bookId 小说ID
      */
     void updateBookPicToLocal(String picUrl, Long bookId);
@@ -198,6 +200,7 @@ public interface BookService {
      * @param userId 用户ID
      * @param page 页码
      * @param pageSize 分页大小
+     * @return 小说集合
      * */
     List<Book> listBookPageByUserId(Long userId, int page, int pageSize);
 

novel-front/src/main/java/com/java2nb/novel/service/FileService.java

@@ -0,0 +1,17 @@
+package com.java2nb.novel.service;
+
+
+/**
+ * @author 11797
+ */
+public interface FileService {
+
+    /**
+     * 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）
+     * @param picSrc 爬取的网络图片路径
+     * @param picSavePath 保存路径
+     * @return 新图片地址
+     * */
+    String transFile(String picSrc, String picSavePath);
+
+}

novel-front/src/main/java/com/java2nb/novel/service/SearchService.java

@@ -0,0 +1,27 @@
+package com.java2nb.novel.service;
+
+
+import com.github.pagehelper.PageInfo;
+import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.search.BookSP;
+
+/**
+ * @author 11797
+ */
+public interface SearchService {
+
+    /**
+     * 导入到es
+     * @param book 小说数据
+     */
+    void importToEs(Book book);
+
+    /**
+     * 搜索
+     * @param params 搜索参数
+     * @param page 当前页码
+     * @param pageSize 每页大小
+     * @return 分页信息
+     */
+    PageInfo searchBook(BookSP params, int page, int pageSize);
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java

@@ -1,6 +1,5 @@
 package com.java2nb.novel.service.impl;
 
-import com.fasterxml.jackson.databind.JavaType;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
@@ -15,21 +14,15 @@ import com.java2nb.novel.mapper.*;
 import com.java2nb.novel.search.BookSP;
 import com.java2nb.novel.service.AuthorService;
 import com.java2nb.novel.service.BookService;
+import com.java2nb.novel.service.FileService;
+import com.java2nb.novel.service.SearchService;
 import com.java2nb.novel.vo.BookCommentVO;
 import com.java2nb.novel.vo.BookSettingVO;
 import com.java2nb.novel.vo.BookVO;
-import com.java2nb.novel.vo.EsBookVO;
-import io.searchbox.client.JestClient;
-import io.searchbox.core.*;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.sort.SortOrder;
 import org.mybatis.dynamic.sql.SortSpecification;
 import org.mybatis.dynamic.sql.render.RenderingStrategies;
 import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
@@ -39,7 +32,6 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import tk.mybatis.orderbyhelper.OrderByHelper;
 
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
@@ -89,7 +81,9 @@ public class BookServiceImpl implements BookService {
 
     private final AuthorService authorService;
 
-    private final JestClient jestClient;
+    private final SearchService searchService;
+
+    private final FileService fileService;
 
 
     @SneakyThrows
@@ -200,139 +194,8 @@ public class BookServiceImpl implements BookService {
 
 
             try {
-                List<EsBookVO> bookList = new ArrayList<>(0);
+                return searchService.searchBook(params,page,pageSize);
 
-                //使用搜索引擎搜索
-                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
-                // 构造查询哪个字段
-                if (StringUtils.isNoneBlank(params.getKeyword())) {
-                    boolQueryBuilder = boolQueryBuilder.must(QueryBuilders.queryStringQuery(params.getKeyword()));
-                }
-
-                // 作品方向
-                if (params.getWorkDirection() != null) {
-                    boolQueryBuilder.filter(QueryBuilders.termQuery("workDirection", params.getWorkDirection()));
-                }
-
-                // 分类
-                if (params.getCatId() != null) {
-                    boolQueryBuilder.filter(QueryBuilders.termQuery("catId", params.getCatId()));
-                }
-                if (params.getBookStatus() != null) {
-                    boolQueryBuilder.filter(QueryBuilders.termQuery("bookStatus", params.getBookStatus()));
-                }
-
-                if (params.getWordCountMin() == null) {
-                    params.setWordCountMin(0);
-                }
-                if (params.getWordCountMax() == null) {
-                    params.setWordCountMax(Integer.MAX_VALUE);
-                }
-
-                boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
-
-                if (params.getUpdateTimeMin() != null) {
-                    boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(params.getUpdateTimeMin()));
-                }
-
-
-
-                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-                searchSourceBuilder.query(boolQueryBuilder);
-
-
-                Count count = new Count.Builder().addIndex("novel").addType("book")
-                        .query(searchSourceBuilder.toString()).build();
-                CountResult results = jestClient.execute(count);
-                Double total = results.getCount();
-
-
-                // 高亮字段
-                HighlightBuilder highlightBuilder = new HighlightBuilder();
-                highlightBuilder.field("authorName");
-                highlightBuilder.field("bookName");
-                highlightBuilder.field("bookDesc");
-                highlightBuilder.field("lastIndexName");
-                highlightBuilder.field("catName");
-                highlightBuilder.preTags("<span style='color:red'>").postTags("</span>");
-                highlightBuilder.fragmentSize(20000);
-                searchSourceBuilder.highlighter(highlightBuilder);
-
-
-                //设置排序
-                if (params.getSort() != null) {
-                    searchSourceBuilder.sort(StringUtil.camelName(params.getSort()), SortOrder.DESC);
-                }
-
-                // 设置分页
-                searchSourceBuilder.from((page - 1) * pageSize);
-                searchSourceBuilder.size(pageSize);
-
-                // 构建Search对象
-                Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex("novel").addType("book").build();
-                log.debug(search.toString());
-                SearchResult result;
-                result = jestClient.execute(search);
-                if (result.isSucceeded()) {
-                    log.debug(result.getJsonString());
-
-                    Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
-                    if (resultMap.get("hits") != null) {
-                        Map hitsMap = (Map) resultMap.get("hits");
-                        if (hitsMap.size() > 0 && hitsMap.get("hits") != null) {
-                            List hitsList = (List) hitsMap.get("hits");
-                            if (hitsList.size() > 0 && result.getSourceAsString() != null) {
-
-                                JavaType jt = new ObjectMapper().getTypeFactory().constructParametricType(ArrayList.class, EsBookVO.class);
-                                bookList = new ObjectMapper().readValue("[" + result.getSourceAsString() + "]", jt);
-
-                                if (bookList != null) {
-                                    for (int i = 0; i < bookList.size(); i++) {
-                                        hitsMap = (Map) hitsList.get(i);
-                                        Map highlightMap = (Map) hitsMap.get("highlight");
-                                        if (highlightMap != null && highlightMap.size() > 0) {
-
-                                            List<String> authorNameList = (List<String>) highlightMap.get("authorName");
-                                            if (authorNameList != null && authorNameList.size() > 0) {
-                                                bookList.get(i).setAuthorName(authorNameList.get(0));
-                                            }
-
-                                            List<String> bookNameList = (List<String>) highlightMap.get("bookName");
-                                            if (bookNameList != null && bookNameList.size() > 0) {
-                                                bookList.get(i).setBookName(bookNameList.get(0));
-                                            }
-
-                                            List<String> bookDescList = (List<String>) highlightMap.get("bookDesc");
-                                            if (bookDescList != null && bookDescList.size() > 0) {
-                                                bookList.get(i).setBookDesc(bookDescList.get(0));
-                                            }
-
-                                            List<String> lastIndexNameList = (List<String>) highlightMap.get("lastIndexName");
-                                            if (lastIndexNameList != null && lastIndexNameList.size() > 0) {
-                                                bookList.get(i).setLastIndexName(lastIndexNameList.get(0));
-                                            }
-
-                                            List<String> catNameList = (List<String>) highlightMap.get("catName");
-                                            if (catNameList != null && catNameList.size() > 0) {
-                                                bookList.get(i).setCatName(catNameList.get(0));
-                                            }
-
-
-                                        }
-                                    }
-
-
-                                }
-                            }
-                        }
-                    }
-
-                    PageInfo<EsBookVO> pageInfo = new PageInfo<>(bookList);
-                    pageInfo.setTotal(total.longValue());
-                    pageInfo.setPageNum(page);
-                    pageInfo.setPageSize(pageSize);
-                    return pageInfo;
-                }
             }catch (Exception e){
                 log.error(e.getMessage(),e);
             }
@@ -479,8 +342,8 @@ public class BookServiceImpl implements BookService {
     }
 
     @Override
-    public void addVisitCount(Long bookId) {
-        bookMapper.addVisitCount(bookId, new Date());
+    public void addVisitCount(Long bookId, Integer visitCount) {
+        bookMapper.addVisitCount(bookId,visitCount);
     }
 
     @Override
@@ -600,14 +463,14 @@ public class BookServiceImpl implements BookService {
     }
 
     @Override
-    public List<Book> queryNetworkPicBooks(Integer limit) {
-        return bookMapper.queryNetworkPicBooks(limit);
+    public List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit) {
+        return bookMapper.queryNetworkPicBooks(localPicPrefix,limit);
     }
 
     @Override
     public void updateBookPicToLocal(String picUrl, Long bookId) {
 
-        picUrl = FileUtil.network2Local(picUrl, picSavePath, Constants.LOCAL_PIC_PREFIX);
+        picUrl = fileService.transFile(picUrl, picSavePath);
 
         bookMapper.update(update(book)
                 .set(BookDynamicSqlSupport.picUrl)

novel-front/src/main/java/com/java2nb/novel/service/impl/FastDfsFileServiceImpl.java

@@ -0,0 +1,60 @@
+package com.java2nb.novel.service.impl;
+
+import com.github.tobato.fastdfs.domain.StorePath;
+import com.github.tobato.fastdfs.service.FastFileStorageClient;
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FilenameUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.io.FileInputStream;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@Slf4j
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "fastDfs")
+public class FastDfsFileServiceImpl implements FileService {
+
+    private final FastFileStorageClient storageClient;
+
+    @Value("${fdfs.webUrl}")
+    private String webUrl;
+
+    @Override
+    public String transFile(String picSrc, String picSavePath) {
+
+        File file;
+        String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+        if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
+            file = new File(picSavePath + filePath);
+        } else {
+            //默认图片不存储
+            return filePath;
+        }
+
+        try {
+            FileInputStream inputStream = new FileInputStream(file);
+            StorePath storePath = storageClient.uploadFile(inputStream, file.length(),
+                    FilenameUtils.getExtension(file.getName()), null);
+            //这里额外加上LOCAL_PIC_PREFIX路径，表明该图片是个人资源，而不是爬虫爬取的网络资源，不需要再次进行转换，
+            // 实际访问时，再通过nginx的rewite指令来重写路径，去掉LOCAL_PIC_PREFIX
+            return webUrl+Constants.LOCAL_PIC_PREFIX+storePath.getFullPath();
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        } finally {
+            //删除
+            file.delete();
+        }
+
+        return "/images/default.gif";
+    }
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/LocalFileServiceImpl.java

@@ -0,0 +1,23 @@
+package com.java2nb.novel.service.impl;
+
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "local")
+public class LocalFileServiceImpl implements FileService {
+
+    @Override
+    public String transFile(String picSrc, String picSavePath){
+
+        return FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+    }
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/OssFileServiceImpl.java

@@ -0,0 +1,76 @@
+package com.java2nb.novel.service.impl;
+
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.model.CannedAccessControlList;
+import com.aliyun.oss.model.CreateBucketRequest;
+import com.aliyun.oss.model.PutObjectRequest;
+import com.aliyun.oss.model.PutObjectResult;
+import com.java2nb.novel.core.config.OssProperties;
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "OSS")
+@Slf4j
+public class OssFileServiceImpl implements FileService {
+
+    private final OssProperties ossProperties;
+
+    @Override
+    public String transFile(String picSrc, String picSavePath) {
+
+        File file;
+        String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+        if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
+            file = new File(picSavePath+filePath);
+        } else {
+            //默认图片不存储
+            return filePath;
+        }
+
+        filePath = filePath.replaceFirst(picSavePath,"");
+
+        filePath = filePath.startsWith("/") ? filePath.replaceFirst("/","") : filePath;
+
+
+        OSSClient ossClient = new OSSClient(ossProperties.getEndpoint(), ossProperties.getKeyId(), ossProperties.getKeySecret());
+        try {
+            //容器不存在，就创建
+            if (!ossClient.doesBucketExist(ossProperties.getBucketName())) {
+                ossClient.createBucket(ossProperties.getBucketName());
+                CreateBucketRequest createBucketRequest = new CreateBucketRequest(ossProperties.getBucketName());
+                createBucketRequest.setCannedACL(CannedAccessControlList.PublicRead);
+                ossClient.createBucket(createBucketRequest);
+            }
+            //上传文件
+            PutObjectResult result = ossClient.putObject(new PutObjectRequest(ossProperties.getBucketName(), filePath, file));
+            //设置权限 这里是公开读
+            ossClient.setBucketAcl(ossProperties.getBucketName(), CannedAccessControlList.PublicRead);
+
+            if(result != null) {
+                return ossProperties.getWebUrl() + "/" + filePath;
+            }
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        } finally {
+            //关闭
+            ossClient.shutdown();
+            file.delete();
+        }
+
+        return "/images/default.gif";
+    }
+
+
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/SearchServiceImpl.java

@@ -0,0 +1,199 @@
+package com.java2nb.novel.service.impl;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.pagehelper.PageInfo;
+import com.java2nb.novel.core.enums.ResponseStatus;
+import com.java2nb.novel.core.exception.BusinessException;
+import com.java2nb.novel.core.utils.StringUtil;
+import com.java2nb.novel.entity.Book;
+import com.java2nb.novel.search.BookSP;
+import com.java2nb.novel.service.SearchService;
+import com.java2nb.novel.vo.EsBookVO;
+import io.searchbox.client.JestClient;
+import io.searchbox.core.*;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.springframework.beans.BeanUtils;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class SearchServiceImpl implements SearchService {
+
+    private final String INDEX = "novel";
+
+    private final String TYPE = "book";
+
+    private final JestClient jestClient;
+
+
+    @Override
+    @SneakyThrows
+    public void importToEs(Book book) {
+        //导入到ES
+        EsBookVO esBookVO = new EsBookVO();
+        BeanUtils.copyProperties(book, esBookVO, "lastIndexUpdateTime");
+        esBookVO.setLastIndexUpdateTime(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(book.getLastIndexUpdateTime()));
+        Index action = new Index.Builder(esBookVO).index(INDEX).type(TYPE).id(book.getId().toString()).build();
+
+        jestClient.execute(action);
+
+
+    }
+
+    @SneakyThrows
+    @Override
+    public PageInfo searchBook(BookSP params, int page, int pageSize) {
+        List<EsBookVO> bookList = new ArrayList<>(0);
+
+        //使用搜索引擎搜索
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        // 构造查询哪个字段
+        if (StringUtils.isNoneBlank(params.getKeyword())) {
+            boolQueryBuilder = boolQueryBuilder.must(QueryBuilders.queryStringQuery(params.getKeyword()));
+        }
+
+        // 作品方向
+        if (params.getWorkDirection() != null) {
+            boolQueryBuilder.filter(QueryBuilders.termQuery("workDirection", params.getWorkDirection()));
+        }
+
+        // 分类
+        if (params.getCatId() != null) {
+            boolQueryBuilder.filter(QueryBuilders.termQuery("catId", params.getCatId()));
+        }
+        if (params.getBookStatus() != null) {
+            boolQueryBuilder.filter(QueryBuilders.termQuery("bookStatus", params.getBookStatus()));
+        }
+
+        if (params.getWordCountMin() == null) {
+            params.setWordCountMin(0);
+        }
+        if (params.getWordCountMax() == null) {
+            params.setWordCountMax(Integer.MAX_VALUE);
+        }
+
+        boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
+
+        if (params.getUpdateTimeMin() != null) {
+            boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(params.getUpdateTimeMin()));
+        }
+
+
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.query(boolQueryBuilder);
+
+
+        Count count = new Count.Builder().addIndex(INDEX).addType(TYPE)
+                .query(searchSourceBuilder.toString()).build();
+        CountResult results = jestClient.execute(count);
+        Double total = results.getCount();
+
+
+        // 高亮字段
+        HighlightBuilder highlightBuilder = new HighlightBuilder();
+        highlightBuilder.field("authorName");
+        highlightBuilder.field("bookName");
+        highlightBuilder.field("bookDesc");
+        highlightBuilder.field("lastIndexName");
+        highlightBuilder.field("catName");
+        highlightBuilder.preTags("<span style='color:red'>").postTags("</span>");
+        highlightBuilder.fragmentSize(20000);
+        searchSourceBuilder.highlighter(highlightBuilder);
+
+
+        //设置排序
+        if (params.getSort() != null) {
+            searchSourceBuilder.sort(StringUtil.camelName(params.getSort()), SortOrder.DESC);
+        }
+
+        // 设置分页
+        searchSourceBuilder.from((page - 1) * pageSize);
+        searchSourceBuilder.size(pageSize);
+
+        // 构建Search对象
+        Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(INDEX).addType(TYPE).build();
+        log.debug(search.toString());
+        SearchResult result;
+        result = jestClient.execute(search);
+        if (result.isSucceeded()) {
+            log.debug(result.getJsonString());
+
+            Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
+            if (resultMap.get("hits") != null) {
+                Map hitsMap = (Map) resultMap.get("hits");
+                if (hitsMap.size() > 0 && hitsMap.get("hits") != null) {
+                    List hitsList = (List) hitsMap.get("hits");
+                    if (hitsList.size() > 0 && result.getSourceAsString() != null) {
+
+                        JavaType jt = new ObjectMapper().getTypeFactory().constructParametricType(ArrayList.class, EsBookVO.class);
+                        bookList = new ObjectMapper().readValue("[" + result.getSourceAsString() + "]", jt);
+
+                        if (bookList != null) {
+                            for (int i = 0; i < bookList.size(); i++) {
+                                hitsMap = (Map) hitsList.get(i);
+                                Map highlightMap = (Map) hitsMap.get("highlight");
+                                if (highlightMap != null && highlightMap.size() > 0) {
+
+                                    List<String> authorNameList = (List<String>) highlightMap.get("authorName");
+                                    if (authorNameList != null && authorNameList.size() > 0) {
+                                        bookList.get(i).setAuthorName(authorNameList.get(0));
+                                    }
+
+                                    List<String> bookNameList = (List<String>) highlightMap.get("bookName");
+                                    if (bookNameList != null && bookNameList.size() > 0) {
+                                        bookList.get(i).setBookName(bookNameList.get(0));
+                                    }
+
+                                    List<String> bookDescList = (List<String>) highlightMap.get("bookDesc");
+                                    if (bookDescList != null && bookDescList.size() > 0) {
+                                        bookList.get(i).setBookDesc(bookDescList.get(0));
+                                    }
+
+                                    List<String> lastIndexNameList = (List<String>) highlightMap.get("lastIndexName");
+                                    if (lastIndexNameList != null && lastIndexNameList.size() > 0) {
+                                        bookList.get(i).setLastIndexName(lastIndexNameList.get(0));
+                                    }
+
+                                    List<String> catNameList = (List<String>) highlightMap.get("catName");
+                                    if (catNameList != null && catNameList.size() > 0) {
+                                        bookList.get(i).setCatName(catNameList.get(0));
+                                    }
+
+
+                                }
+                            }
+
+
+                        }
+                    }
+                }
+            }
+
+            PageInfo<EsBookVO> pageInfo = new PageInfo<>(bookList);
+            pageInfo.setTotal(total.longValue());
+            pageInfo.setPageNum(page);
+            pageInfo.setPageSize(pageSize);
+            return pageInfo;
+        }
+       throw new BusinessException(ResponseStatus.ES_SEARCH_FAIL);
+    }
+}

novel-front/src/main/resources/application-fastdfs.yml

@@ -0,0 +1,16 @@
+#FastDfs的配置	====================================
+#读取inputsream阻塞时间
+fdfs:
+  connect-timeout: 600
+  so-timeout: 1500
+  #tracker地址
+  trackerList: 127.0.0.1:22122
+  #缩略图配置
+  thumbImage:
+    height: 150
+    width: 150
+  #通过nginx 访问地址
+  webUrl: http://127.0.0.1/
+  #获取连接池最大数量
+  pool.max-total: 200
+

novel-front/src/main/resources/application-oss.yml

@@ -0,0 +1,9 @@
+#OSS
+novel:
+  file:
+    endpoint: oss-cn-shenzhen.aliyuncs.com #不同的服务器，地址不同
+    key-id: dhjjrue6767778878 #去OSS控制台获取
+    key-secret: dssdkkrkelrkellk44554 #去OSS控制台获取
+    bucket-name: novel #这个自己创建bucket时的命名，控制台创建也行，代码创建也行
+    file-host: pic #文件路径
+    web-url: https://xxyimg.oss-cn-hangzhou.aliyuncs.com #OSS文件的web访问地址

novel-front/src/main/resources/application.yml

@@ -4,7 +4,19 @@ server:
 spring:
   profiles:
     active: dev
-    include: alipay
+    include: alipay,oss,fastdfs
+
+
+  rabbitmq:
+    enable: 0
+    host: 127.0.0.1
+    username: guest
+    password: guest
+    virtual-host: /novel-plus
+    template:
+      # 缺省的交换机名称，此处配置后，发送消息如果不指定交换机就会使用这个
+      exchange: novel.exchange
+    publisher-confirms: false
 
 
   elasticsearch:
@@ -13,7 +25,9 @@ spring:
     jest:
       uris: http://127.0.0.1:9200
 
-
+redisson:
+  singleServerConfig:
+    address: 127.0.0.1:6379
 
 jwt:
   secret: novel!#20191230

novel-front/src/main/resources/mybatis/mapping/BookMapper.xml

@@ -35,7 +35,7 @@
     </select>
 
     <update id="addVisitCount" >
-        update book set visit_count = visit_count + 1 , update_time = #{date}
+        update book set visit_count = visit_count + ${visitCount}
         where id = #{bookId}
     </update>
 
@@ -56,6 +56,7 @@
         select
         id,pic_url from book
         where pic_url like 'http%'
+        and pic_url not like concat('%',#{localPicPrefix},'%')
         limit #{limit}
     </select>
 

novel-front/src/main/resources/templates/mobile/book/book_ranking.html

@@ -179,9 +179,13 @@
                     for (var i = 0; i < bookList.length; i++) {
                         var book = bookList[i];
 
-                        var end = book.bookDesc.indexOf("<");
+                        /*var end = book.bookDesc.indexOf("<");
                         if(end != -1) {
                             book.bookDesc = book.bookDesc.substring(0,end);
+                        }*/
+
+                        if(book.bookDesc){
+                            book.bookDesc = book.bookDesc.replace(/<[^>]+>/g,"").replace(/\s+/g,"").replace(/&nbsp;/g,"");
                         }
 
                         bookListHtml += ("<div class=\"layui-row\" style=\"margin-bottom:10px;padding:10px;background: #f2f2f2\">\n" +
@@ -202,7 +206,7 @@
                             "            </a>\n" +
                             "            <div style=\"margin-top: 5px;color: #4c6978;\">类别："+book.catName+"</div>\n" +
                             "            <div style=\"margin-top: 5px;color: #4c6978;\">状态："+(book.bookStatus==0?'连载':'完结')+"</div>\n" +
-                            "            <div style=\"margin-top: 5px;color: #4c6978;\">更新：<i>"+book.lastIndexUpdateTime+"</i>\n" +
+                            "            <div style=\"margin-top: 5px;color: #4c6978;\">更新：<i>"+book.lastIndexUpdateTime.substr(0,11)+"</i>\n" +
                             "            </div>\n" +
                             "            <div style=\"margin-top: 5px;color: #4c6978;\">简介："+(book.bookDesc?(book.bookDesc.length>20?(book.bookDesc.substr(0,20)+"..."):book.bookDesc):book.bookDesc)+"</div>\n" +
                             "\n" +

novel-front/src/main/resources/templates/mobile/index.html

@@ -276,6 +276,10 @@
                 for (var i = 0; i < 6; i++) {
                     var hotRecBook = hotRecBooks[i];
 
+                    if(hotRecBook.bookDesc){
+                        hotRecBook.bookDesc = hotRecBook.bookDesc.replace(/<[^>]+>/g,"").replace(/\s+/g,"");
+                    }
+
                     hotRecBooksHtml += ("<div style=\"margin-bottom: 5px\" class=\"layui-col-xs12 layui-col-sm6 layui-col-md4 layui-col-lg4\">\n" +
                         "                        <a href=\"/book/"+hotRecBook.bookId+".html\">\n" +
                         "                            <div class=\"layui-col-xs5 layui-col-sm4 layui-col-md4 layui-col-lg4\" >\n" +
@@ -323,9 +327,8 @@
                 for (var i = 0; i < 10; i++) {
 
                     var updateRankBook = updateRankBooks[i];
-                    var end = updateRankBook.bookDesc.indexOf("<");
-                    if(end != -1) {
-                        updateRankBook.bookDesc = updateRankBook.bookDesc.substring(0,end);
+                    if(updateRankBook.bookDesc){
+                        updateRankBook.bookDesc = updateRankBook.bookDesc.replace(/<[^>]+>/g,"").replace(/\s+/g,"");
                     }
 
                         updateRankBookHtml += ("<div style=\"padding-bottom: 30px\"\n" +

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.java2nb</groupId>
     <artifactId>novel</artifactId>
-    <version>2.1.1</version>
+    <version>2.5.1</version>
     <modules>
         <module>novel-common</module>
         <module>novel-front</module>
@@ -38,6 +38,13 @@
         <jjwt.version>0.9.0</jjwt.version>
         <elasticsearch.version>6.2.2</elasticsearch.version>
         <jest.version>6.3.1</jest.version>
+        <redis.version>1.4.1.RELEASE</redis.version>
+        <redisson.version>3.12.5</redisson.version>
+        <sharding.jdbc.version>3.0.0</sharding.jdbc.version>
+        <aliyun-sdk-oss.version>2.4.0</aliyun-sdk-oss.version>
+        <commons-fileupload.version>1.3.1</commons-fileupload.version>
+        <fastdfs-client.version>1.26.1-RELEASE</fastdfs-client.version>
+        <alipay-sdk-java.version>4.9.153.ALL</alipay-sdk-java.version>
     </properties>
 
     <dependencyManagement>

sql/20200608.sql

@@ -0,0 +1,2 @@
+INSERT INTO `crawl_source` (`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES
+(6, '新笔趣阁', '{\n	\"bookListUrl\": \"http://www.xbiquge.la/fenlei/{catId}_{page}.html\",\n	\"catIdRule\": {\n		\"catId1\": \"1\",\n		\"catId2\": \"2\",\n		\"catId3\": \"3\",\n		\"catId4\": \"4\",\n		\"catId5\": \"6\",\n		\"catId6\": \"5\"\n	},\n	\"bookIdPatten\": \"<a\\\\s+href=\\\"http://www.xbiquge.la/(\\\\d+/\\\\d+)/\\\"\\\\s+target=\\\"_blank\\\">\",\n	\"pagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">(\\\\d+)/\\\\d+</em>\",\n	\"totalPagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">\\\\d+/(\\\\d+)</em>\",\n	\"bookDetailUrl\": \"http://www.xbiquge.la/{bookId}/\",\n	\"bookNamePatten\": \"<h1>([^/]+)</h1>\",\n	\"authorNamePatten\": \"者：([^/]+)</p>\",\n	\"picUrlPatten\": \"src=\\\"(http://www.xbiquge.la/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n	\"bookStatusRule\": {},\n	\"descStart\": \"<div id=\\\"intro\\\">\",\n	\"descEnd\": \"</div>\",\n	\"upadateTimePatten\": \"<p>最后更新：(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)</p>\",\n	\"upadateTimeFormatPatten\": \"yyyy-MM-dd HH:mm:ss\",\n	\"bookIndexUrl\": \"http://www.xbiquge.la/{bookId}/\",\n	\"indexIdPatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/(\\\\d+)\\\\.html\'\\\\s+>[^/]+</a>\",\n	\"indexNamePatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/\\\\d+\\\\.html\'\\\\s+>([^/]+)</a>\",\n	\"bookContentUrl\": \"http://www.xbiquge.la/{bookId}/{indexId}.html\",\n	\"contentStart\": \"<div id=\\\"content\\\">\",\n	\"contentEnd\": \"<p>\"\n}', 0, '2020-05-23 22:46:58', '2020-05-23 22:46:58');

sql/novel_plus.sql

@@ -1691,4 +1691,122 @@ INSERT INTO `user_read_history` VALUES ('113', '1255060328322027520', '125467697
 INSERT INTO `user_read_history` VALUES ('117', '1255060328322027520', '1254946661743603712', '1254946914001629184', '2020-04-30 19:37:09', '2020-04-30 19:37:09');
 INSERT INTO `user_read_history` VALUES ('118', '1255060328322027520', '1254957312633352192', '3335449', '2020-04-30 19:37:36', '2020-04-30 19:37:36');
 
+
+
+DROP TABLE IF EXISTS `book_content0`;
+CREATE TABLE `book_content0` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1155 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content1
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content1`;
+CREATE TABLE `book_content1` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=406 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content2
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content2`;
+CREATE TABLE `book_content2` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1222 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content3
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content3`;
+CREATE TABLE `book_content3` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=410 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content4
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content4`;
+CREATE TABLE `book_content4` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1188 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content5
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content5`;
+CREATE TABLE `book_content5` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=416 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content6
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content6`;
+CREATE TABLE `book_content6` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1180 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content7
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content7`;
+CREATE TABLE `book_content7` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=404 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content8
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content8`;
+CREATE TABLE `book_content8` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=1134 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
+
+-- ----------------------------
+-- Table structure for book_content9
+-- ----------------------------
+DROP TABLE IF EXISTS `book_content9`;
+CREATE TABLE `book_content9` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
+  `content` mediumtext COMMENT '小说章节内容',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
 UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n	\"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n	\"catIdRule\": {\n		\"catId1\": \"1\",\n		\"catId2\": \"2\",\n		\"catId3\": \"3\",\n		\"catId4\": \"4\",\n		\"catId5\": \"7\",\n		\"catId6\": \"6\",\n		\"catId7\": \"8\"\n	},\n	\"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n	\"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n	\"totalPagePatten\": \"第\\\\d+/(\\\\d+)页\",\n	\"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n	\"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n	\"authorNamePatten\": \"<p>作者：([^/]+)</p>\",\n	\"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n	\"statusPatten\": \"<p>状态：([^/]+)</p>\",\n	\"bookStatusRule\": {\n		\"连载中\": 0,\n		\"完本\": 1\n	},\n	\"descStart\": \"<div class=\\\"intro_info\\\">\",\n	\"descEnd\": \"最新章节推荐地址\",\n	\"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n	\"bookIndexStart\": \"<dt>《\",\n	\"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n	\"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n	\"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n	\"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n	\"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;