Compare commits

..

1 Commits

Author SHA1 Message Date
xxy
b1db37bb3d v2.2.0发布 2020-05-26 07:50:39 +08:00
36 changed files with 199 additions and 998 deletions

View File

@ -1,15 +1,7 @@
[![index]( https://i.loli.net/2020/06/01/qF1uBMtswTIPJKx.jpg )]( https://www.eve.bz/cart.php?gid=11 )
[![index](./assets/热门云产品1040.100.jpg)](https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console)
# 小说精品屋-plus
#### 新项目小说精品屋-微服务版
基于小说精品屋-plus构建的Spring Cloud 微服务小说门户平台可用于学习和商用
Gitee仓库地址 https://gitee.com/xiongxyang/novel-cloud
GitHub仓库地址 https://github.com/201206030/novel-cloud
#### 演示地址
[点击前往](http://www.java2nb.com)(前台门户)
@ -48,31 +40,7 @@ novel-plus -- 父工程
```
#### 技术选型
| 技术 | 说明
| -------------------- | ---------------------------
| SpringBoot | Spring应用快速开发脚手架
| MyBatis | 持久层ORM框架
| MyBatis Dynamic SQL | Mybatis动态sql
| PageHelper | MyBatis分页插件
| MyBatisGenerator | 持久层代码生成插件
| Sharding-Jdbc | 代码层分库分表中间件
| JJWT | JWT登录支持
| SpringSecurity | 安全框架
| Shiro | 安全框架
| Ehcache | Java进程内缓存框架(默认缓存)
| Redis | 分布式缓存(缓存替换方案默认关闭一行配置开启)
| ElasticSearch | 搜索引擎(搜索增强方案默认关闭一行配置开启)
| RabbitMq | 消息队列(流量削峰默认关闭一行配置开启)
| OSS | 阿里云对象存储服务(图片存储方式之一一行配置即可切换)
| FastDfs |开源轻量级分布式文件系统(图片存储方式之一一行配置即可切换)
| Redisson | 实现分布式锁
| Lombok | 简化对象封装工具
| Docker | 应用容器引擎
| Mysql | 数据库服务
| Thymeleaf | 模板引擎
| Layui | 前端UI
Springboot+Mybatis+Mysql+ElasticSearch+Ehcache+Thymeleaf+Layui
#### PC站截图

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>2.5.1</version>
<version>2.2.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -57,22 +57,6 @@
<version>${mybatis.version}</version>
</dependency>
<!-- 分库分表-->
<!-- sharding jdbc依赖 -->
<dependency>
<groupId>io.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-boot-starter</artifactId>
<version>${sharding.jdbc.version}</version>
</dependency>
<dependency>
<groupId>io.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-namespace</artifactId>
<version>${sharding.jdbc.version}</version>
</dependency>
<!--分页助手启动器-->
<dependency>
<groupId>com.github.pagehelper</groupId>

View File

@ -55,14 +55,4 @@ public interface CacheKey {
* 搜索引擎转换锁
* */
String ES_TRANS_LOCK = "esTransLock";
/**
* 上一次搜索引擎是否更新过小说点击量
* */
String ES_IS_UPDATE_VISIT = "esIsUpdateVisit";
/**
* 累积的小说点击量
* */
String BOOK_ADD_VISIT_COUNT = "bookAddVisitCount";
}

View File

@ -58,16 +58,13 @@ public enum ResponseStatus {
, BOOKNAME_EXISTS(4003,"已发布过同名小说!")
,
/**
* 搜索引擎相关错误
* */
ES_SEARCH_FAIL(9001,"密码错误!"),
/**
* 其他通用错误
* */
PASSWORD_ERROR(88001,"密码错误!");
private int code;
private String msg;

View File

@ -24,7 +24,7 @@ public final class BookContentDynamicSqlSupport {
public final SqlColumn<Long> indexId = column("index_id", JDBCType.BIGINT);
public final SqlColumn<String> content = column("content", JDBCType.VARCHAR);
public final SqlColumn<String> content = column("content", JDBCType.LONGVARCHAR);
public BookContent() {
super("book_content");

View File

@ -63,7 +63,7 @@ public interface BookContentMapper {
@Results(id="BookContentResult", value = {
@Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
@Result(column="index_id", property="indexId", jdbcType=JdbcType.BIGINT),
@Result(column="content", property="content", jdbcType=JdbcType.VARCHAR)
@Result(column="content", property="content", jdbcType=JdbcType.LONGVARCHAR)
})
List<BookContent> selectMany(SelectStatementProvider selectStatement);

View File

@ -1,8 +1,11 @@
spring:
profiles:
include: [common]
main:
allow-bean-definition-overriding: true
datasource:
url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: test123456
driver-class-name: com.mysql.cj.jdbc.Driver
#Redis服务器IP
redis:
host: 127.0.0.1
@ -22,48 +25,6 @@ spring:
min-idle: 0
#连接超时时间毫秒
timeout: 30000
datasource:
url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: test123456
driver-class-name: com.mysql.cj.jdbc.Driver
####使用shardingJdbc时
####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误
##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR
sharding:
jdbc:
datasource:
names: ds0 #,ds1
ds0:
type: com.zaxxer.hikari.HikariDataSource
driver-class-name: com.mysql.cj.jdbc.Driver
jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: test123456
# ds1:
# type: com.alibaba.druid.pool.DruidDataSource
# driver-class-name: com.mysql.jdbc.Driver
# url: jdbc:mysql://localhost:3306/novel_plus2
# username: root
# password: test123456
config:
sharding:
props:
sql.show: true
tables:
book_content: #book_content表
key-generator-column-name: id #主键
actual-data-nodes: ds${0}.book_content${0..9} #数据节点
# database-strategy: #分库策略
# inline:
# sharding-column: book_id
# algorithm-expression: ds${book_id % 10}
table-strategy: #分表策略
inline:
shardingColumn: index_id
algorithm-expression: book_content${index_id % 10}
@ -72,8 +33,7 @@ sharding:
pic:
save:
type: 2 #图片保存方式, 1不保存使用爬取的网络图片 2保存在自己的存储介质
storage: local #存储介质local本地OSS阿里云对象存储fastDfs分布式文件系统
type: 2 #图片保存方式, 1不保存使用网络图片 2本地保存
path: /var/pic #图片保存路径

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>2.5.1</version>
<version>2.2.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -25,8 +25,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
</dependencies>
<build>

View File

@ -96,12 +96,8 @@ public class CrawlParser {
String desc = bookDetailHtml.substring(bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length());
desc = desc.substring(0, desc.indexOf(ruleBean.getDescEnd()));
//过滤掉简介中的特殊标签
desc = desc.replaceAll("<a[^<]+</a>","")
.replaceAll("<font[^<]+</font>","")
.replaceAll("<p>\\s*</p>","")
.replaceAll("<p>","")
.replaceAll("</p>","<br/>");
//过滤掉简介中的a标签
desc = desc.replaceAll("<a[^<]+</a>","");
//设置书籍简介
book.setBookDesc(desc);
if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {

View File

@ -61,7 +61,7 @@ public class StarterListener implements ServletContextListener {
Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
//这里只做老书更新
book.setId(needUpdateBook.getId());
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
//本地图片则不更新
book.setPicUrl(null);
}

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>2.5.1</version>
<version>2.2.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -27,11 +27,6 @@
<version>${jjwt.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<dependency>
<groupId>io.searchbox</groupId>
@ -39,45 +34,16 @@
<version>${jest.version}</version>
</dependency>
<!--aliyunOSS-->
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>${aliyun-sdk-oss.version}</version>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>${commons-fileupload.version}</version>
</dependency>
<!--FastDfs-->
<dependency>
<groupId>com.github.tobato</groupId>
<artifactId>fastdfs-client</artifactId>
<version>${fastdfs-client.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<!--引入redisson分布式锁-->
<!-- <dependency>
<groupId>org.redisson</groupId>
<artifactId>redisson-spring-boot-starter</artifactId>
<version>${redisson.version}</version>
</dependency>-->
<dependency>
<groupId>com.alipay.sdk</groupId>
<artifactId>alipay-sdk-java</artifactId>
<version>${alipay-sdk-java.version}</version>
<version>4.9.153.ALL</version>
</dependency>
</dependencies>

View File

@ -1,13 +1,10 @@
package com.java2nb.novel;
import com.github.tobato.fastdfs.FdfsClientConfig;
import io.shardingsphere.shardingjdbc.spring.boot.SpringBootConfiguration;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
@ -21,7 +18,6 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
@EnableScheduling
@EnableCaching
@MapperScan(basePackages = {"com.java2nb.novel.mapper"})
@Import(FdfsClientConfig.class)
public class FrontNovelApplication {
public static void main(String[] args) {

View File

@ -4,14 +4,14 @@ import com.github.pagehelper.PageInfo;
import com.java2nb.novel.core.bean.ResultBean;
import com.java2nb.novel.core.bean.UserDetails;
import com.java2nb.novel.core.enums.ResponseStatus;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.BookComment;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.search.BookSP;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.vo.BookVO;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@ -19,6 +19,7 @@ import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
@ -32,11 +33,6 @@ public class BookController extends BaseController{
private final BookService bookService;
private final RabbitTemplate rabbitTemplate;
@Value("${spring.rabbitmq.enable}")
private Integer enableMq;
/**
* 查询首页小说设置列表数据
@ -109,11 +105,7 @@ public class BookController extends BaseController{
* */
@PostMapping("addVisitCount")
public ResultBean addVisitCount(Long bookId){
if(enableMq == 1) {
rabbitTemplate.convertAndSend("ADD-BOOK-VISIT-EXCHANGE", null, bookId);
}else {
bookService.addVisitCount(bookId, 1);
}
bookService.addVisitCount(bookId);
return ResultBean.ok();
}

View File

@ -1,13 +0,0 @@
package com.java2nb.novel.core.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableMBeanExport;
import org.springframework.jmx.support.RegistrationPolicy;
/**
* @author 11797
*/
@Configuration
@EnableMBeanExport(registration= RegistrationPolicy.IGNORE_EXISTING)
public class FdfsConfiguration {
}

View File

@ -1,30 +0,0 @@
package com.java2nb.novel.core.config;
import lombok.Data;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* @author 11797
*/
@Data
@Component
@ConfigurationProperties(prefix="novel.file")
public class OssProperties{
private String endpoint;
private String keyId;
private String keySecret;
private String fileHost;
private String bucketName;
private String webUrl;
}

View File

@ -1,59 +0,0 @@
package com.java2nb.novel.core.config;
import org.springframework.amqp.core.*;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author 11797
*/
@Configuration
@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "enable", havingValue = "1")
public class RabbitConfig {
/**
* 更新数据库队列
*/
@Bean
public Queue updateDbQueue() {
return new Queue("UPDATE-DB-QUEUE", true);
}
/**
* 更新数据库队列
*/
@Bean
public Queue updateEsQueue() {
return new Queue("UPDATE-ES-QUEUE", true);
}
/**
* 增加点击量交换机
*/
@Bean
public FanoutExchange addVisitExchange() {
return new FanoutExchange("ADD-BOOK-VISIT-EXCHANGE");
}
/**
* 更新搜索引擎队列绑定到增加点击量交换机中
*/
@Bean
public Binding updateEsBinding() {
return BindingBuilder.bind(updateEsQueue()).to(addVisitExchange());
}
/**
* 更新数据库绑定到增加点击量交换机中
*/
@Bean
public Binding updateDbBinding() {
return BindingBuilder.bind(updateDbQueue()).to(addVisitExchange());
}
}

View File

@ -1,107 +0,0 @@
package com.java2nb.novel.core.listener;
import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.SearchService;
import com.java2nb.novel.vo.EsBookVO;
import com.rabbitmq.client.Channel;
import io.searchbox.client.JestClient;
import io.searchbox.core.Index;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.core.Message;
import org.springframework.beans.BeanUtils;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import java.text.SimpleDateFormat;
/**
* @author 11797
*/
@Component
@Slf4j
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "enable", havingValue = "1")
public class BookVisitAddListener {
private final BookService bookService;
private final CacheService cacheService;
private final SearchService searchService;
// private final RedissonClient redissonClient;
/**
* 更新数据库
* 流量削峰每本小说累积10个点击更新一次
*/
@SneakyThrows
@RabbitListener(queues = {"UPDATE-DB-QUEUE"})
public void updateDb(Long bookId, Channel channel, Message message) {
log.debug("收到更新数据库消息:" + bookId);
Thread.sleep(1000 * 2);
//TODO 操作共享资源visitCount集群环境下有线程安全问题引入Redisson框架实现分布式锁
//RLock lock = redissonClient.getLock("visitCount");
//lock.lock();
//目前visitCount不重要数据可丢失暂不实现分布式锁
Integer visitCount = (Integer) cacheService.getObject(CacheKey.BOOK_ADD_VISIT_COUNT+bookId);
if(visitCount == null){
visitCount = 0 ;
}
cacheService.setObject(CacheKey.BOOK_ADD_VISIT_COUNT+bookId,++visitCount);
if(visitCount >= Constants.ADD_MAX_VISIT_COUNT) {
bookService.addVisitCount(bookId,visitCount);
cacheService.del(CacheKey.BOOK_ADD_VISIT_COUNT+bookId);
}
//TODO 操作共享资源visitCount集群环境下有线程安全问题引入Redisson框架实现分布式锁
//lock.unlock();
}
/**
* 更新搜索引擎
* 流量削峰每本小说1个小时更新一次
*/
@RabbitListener(queues = {"UPDATE-ES-QUEUE"})
public void updateEs(Long bookId, Channel channel, Message message) {
log.debug("收到更新搜索引擎消息:" + bookId);
if (cacheService.get(CacheKey.ES_IS_UPDATE_VISIT + bookId) == null) {
cacheService.set(CacheKey.ES_IS_UPDATE_VISIT + bookId, "1", 60 * 60);
try {
Thread.sleep(1000 * 5);
Book book = bookService.queryBookDetail(bookId);
searchService.importToEs(book);
}catch (Exception e){
cacheService.del(CacheKey.ES_IS_UPDATE_VISIT + bookId);
log.error("更新搜索引擎失败"+bookId);
}
}
}
}

View File

@ -5,7 +5,6 @@ import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.utils.BeanUtil;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.SearchService;
import com.java2nb.novel.vo.EsBookVO;
import io.searchbox.client.JestClient;
import io.searchbox.core.DocumentResult;
@ -38,20 +37,16 @@ public class BookToEsSchedule {
private final CacheService cacheService;
private final SearchService searchService;
private final JestClient jestClient;
/**
* 1钟导入一次
* 10秒钟导入一次
*/
@Scheduled(fixedRate = 1000 * 60)
@Scheduled(fixedRate = 1000 * 10)
public void saveToEs() {
//TODO 引入Redisson框架实现分布式锁
//可以重复更新,只是效率可能略有降低,所以暂不实现分布式锁
if (cacheService.get(CacheKey.ES_TRANS_LOCK) == null) {
cacheService.set(CacheKey.ES_TRANS_LOCK, "1", 60 * 20);
cacheService.set(CacheKey.ES_TRANS_LOCK, "1", 60 * 5);
try {
//查询需要更新的小说
Date lastDate = (Date) cacheService.getObject(CacheKey.ES_LAST_UPDATE_TIME);
@ -62,9 +57,17 @@ public class BookToEsSchedule {
List<Book> books = bookService.queryBookByUpdateTimeByPage(lastDate, 100);
for (Book book : books) {
searchService.importToEs(book);
//导入到ES
EsBookVO esBookVO = new EsBookVO();
BeanUtils.copyProperties(book, esBookVO, "lastIndexUpdateTime");
esBookVO.setLastIndexUpdateTime(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(book.getLastIndexUpdateTime()));
Index action = new Index.Builder(esBookVO).index("novel").type("book").id(book.getId().toString()).build();
jestClient.execute(action);
lastDate = book.getUpdateTime();
Thread.sleep(5000);
Thread.sleep(1000);
}

View File

@ -1,6 +1,5 @@
package com.java2nb.novel.core.schedule;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.service.BookService;
import lombok.RequiredArgsConstructor;
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
import java.util.List;
/**
* 将爬取的网络图片转存为自己的存储介质本地OSSfastDfs任务
* 网络图片转存本地任务
*
* @author Administrator
*/
@ -22,7 +21,7 @@ import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class CrawlPicTransSchedule {
public class Network2LocalPicSchedule {
private final BookService bookService;
@ -42,7 +41,7 @@ public class CrawlPicTransSchedule {
log.info("Network2LocalPicSchedule。。。。。。。。。。。。");
List<Book> networkPicBooks = bookService.queryNetworkPicBooks(Constants.LOCAL_PIC_PREFIX,100);
List<Book> networkPicBooks = bookService.queryNetworkPicBooks(100);
for (Book book : networkPicBooks) {
bookService.updateBookPicToLocal(book.getPicUrl(), book.getId());
//3秒钟转化一张图片10分钟转化200张

View File

@ -12,7 +12,7 @@ public class Constants {
public static final String TEMPLATE_PATH_PREFIX_KEY = "templatePathPrefixKey";
/**
* 保存图片到自己的存储介质路径前缀
* 本地图片保存前缀
* */
public static final String LOCAL_PIC_PREFIX = "/localPic/";
@ -30,9 +30,4 @@ public class Constants {
* 首页设置的小说数量
* */
public static final int INDEX_BOOK_SETTING_NUM = 32;
/**
* 累积的最大点击量
* */
public static final Integer ADD_MAX_VISIT_COUNT = 10;
}

View File

@ -16,13 +16,13 @@ public interface FrontBookMapper extends BookMapper {
List<BookVO> searchByPage(BookSP params);
void addVisitCount(@Param("bookId") Long bookId, @Param("visitCount") Integer visitCount);
void addVisitCount(@Param("bookId") Long bookId, @Param("date") Date date);
List<Book> listRecBookByCatId(@Param("catId") Integer catId);
void addCommentCount(@Param("bookId") Long bookId);
List<Book> queryNetworkPicBooks(@Param("localPicPrefix") String localPicPrefix, @Param("limit") Integer limit);
List<Book> queryNetworkPicBooks(@Param("limit") Integer limit);
/**
* 按评分随机查询小说集合

View File

@ -112,8 +112,8 @@ public interface BookService {
/**
* 增加点击次数
* @param bookId 书籍ID
* @param visitCount*/
void addVisitCount(Long bookId, Integer visitCount);
* */
void addVisitCount(Long bookId);
/**
* 查询章节数
@ -180,17 +180,15 @@ public interface BookService {
/**
* 查询网络图片的小说
*
* @param localPicPrefix
* @param limit 查询条数
* @return 返回小说集合
* */
List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit);
List<Book> queryNetworkPicBooks(Integer limit);
/**
* 更新爬取的小说网络图片到自己的存储介质本地、OSS、fastDfs
* @param picUrl 爬取的网络图片路径
* 更新小说网络图片到本地
* @param picUrl 网络图片路径
* @param bookId 小说ID
*/
void updateBookPicToLocal(String picUrl, Long bookId);
@ -200,7 +198,6 @@ public interface BookService {
* @param userId 用户ID
* @param page 页码
* @param pageSize 分页大小
* @return 小说集合
* */
List<Book> listBookPageByUserId(Long userId, int page, int pageSize);

View File

@ -1,17 +0,0 @@
package com.java2nb.novel.service;
/**
* @author 11797
*/
public interface FileService {
/**
* 将爬取的网络图片转存为自己的存储介质本地、OSS、fastDfs
* @param picSrc 爬取的网络图片路径
* @param picSavePath 保存路径
* @return 新图片地址
* */
String transFile(String picSrc, String picSavePath);
}

View File

@ -1,27 +0,0 @@
package com.java2nb.novel.service;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.search.BookSP;
/**
* @author 11797
*/
public interface SearchService {
/**
* 导入到es
* @param book 小说数据
*/
void importToEs(Book book);
/**
* 搜索
* @param params 搜索参数
* @param page 当前页码
* @param pageSize 每页大小
* @return 分页信息
*/
PageInfo searchBook(BookSP params, int page, int pageSize);
}

View File

@ -1,5 +1,6 @@
package com.java2nb.novel.service.impl;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
@ -14,15 +15,21 @@ import com.java2nb.novel.mapper.*;
import com.java2nb.novel.search.BookSP;
import com.java2nb.novel.service.AuthorService;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.FileService;
import com.java2nb.novel.service.SearchService;
import com.java2nb.novel.vo.BookCommentVO;
import com.java2nb.novel.vo.BookSettingVO;
import com.java2nb.novel.vo.BookVO;
import com.java2nb.novel.vo.EsBookVO;
import io.searchbox.client.JestClient;
import io.searchbox.core.*;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.mybatis.dynamic.sql.SortSpecification;
import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
@ -32,6 +39,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import tk.mybatis.orderbyhelper.OrderByHelper;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@ -81,9 +89,7 @@ public class BookServiceImpl implements BookService {
private final AuthorService authorService;
private final SearchService searchService;
private final FileService fileService;
private final JestClient jestClient;
@SneakyThrows
@ -194,8 +200,139 @@ public class BookServiceImpl implements BookService {
try {
return searchService.searchBook(params,page,pageSize);
List<EsBookVO> bookList = new ArrayList<>(0);
//使用搜索引擎搜索
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// 构造查询哪个字段
if (StringUtils.isNoneBlank(params.getKeyword())) {
boolQueryBuilder = boolQueryBuilder.must(QueryBuilders.queryStringQuery(params.getKeyword()));
}
// 作品方向
if (params.getWorkDirection() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("workDirection", params.getWorkDirection()));
}
// 分类
if (params.getCatId() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("catId", params.getCatId()));
}
if (params.getBookStatus() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("bookStatus", params.getBookStatus()));
}
if (params.getWordCountMin() == null) {
params.setWordCountMin(0);
}
if (params.getWordCountMax() == null) {
params.setWordCountMax(Integer.MAX_VALUE);
}
boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
if (params.getUpdateTimeMin() != null) {
boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(params.getUpdateTimeMin()));
}
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQueryBuilder);
Count count = new Count.Builder().addIndex("novel").addType("book")
.query(searchSourceBuilder.toString()).build();
CountResult results = jestClient.execute(count);
Double total = results.getCount();
// 高亮字段
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.field("authorName");
highlightBuilder.field("bookName");
highlightBuilder.field("bookDesc");
highlightBuilder.field("lastIndexName");
highlightBuilder.field("catName");
highlightBuilder.preTags("<span style='color:red'>").postTags("</span>");
highlightBuilder.fragmentSize(20000);
searchSourceBuilder.highlighter(highlightBuilder);
//设置排序
if (params.getSort() != null) {
searchSourceBuilder.sort(StringUtil.camelName(params.getSort()), SortOrder.DESC);
}
// 设置分页
searchSourceBuilder.from((page - 1) * pageSize);
searchSourceBuilder.size(pageSize);
// 构建Search对象
Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex("novel").addType("book").build();
log.debug(search.toString());
SearchResult result;
result = jestClient.execute(search);
if (result.isSucceeded()) {
log.debug(result.getJsonString());
Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
if (resultMap.get("hits") != null) {
Map hitsMap = (Map) resultMap.get("hits");
if (hitsMap.size() > 0 && hitsMap.get("hits") != null) {
List hitsList = (List) hitsMap.get("hits");
if (hitsList.size() > 0 && result.getSourceAsString() != null) {
JavaType jt = new ObjectMapper().getTypeFactory().constructParametricType(ArrayList.class, EsBookVO.class);
bookList = new ObjectMapper().readValue("[" + result.getSourceAsString() + "]", jt);
if (bookList != null) {
for (int i = 0; i < bookList.size(); i++) {
hitsMap = (Map) hitsList.get(i);
Map highlightMap = (Map) hitsMap.get("highlight");
if (highlightMap != null && highlightMap.size() > 0) {
List<String> authorNameList = (List<String>) highlightMap.get("authorName");
if (authorNameList != null && authorNameList.size() > 0) {
bookList.get(i).setAuthorName(authorNameList.get(0));
}
List<String> bookNameList = (List<String>) highlightMap.get("bookName");
if (bookNameList != null && bookNameList.size() > 0) {
bookList.get(i).setBookName(bookNameList.get(0));
}
List<String> bookDescList = (List<String>) highlightMap.get("bookDesc");
if (bookDescList != null && bookDescList.size() > 0) {
bookList.get(i).setBookDesc(bookDescList.get(0));
}
List<String> lastIndexNameList = (List<String>) highlightMap.get("lastIndexName");
if (lastIndexNameList != null && lastIndexNameList.size() > 0) {
bookList.get(i).setLastIndexName(lastIndexNameList.get(0));
}
List<String> catNameList = (List<String>) highlightMap.get("catName");
if (catNameList != null && catNameList.size() > 0) {
bookList.get(i).setCatName(catNameList.get(0));
}
}
}
}
}
}
}
PageInfo<EsBookVO> pageInfo = new PageInfo<>(bookList);
pageInfo.setTotal(total.longValue());
pageInfo.setPageNum(page);
pageInfo.setPageSize(pageSize);
return pageInfo;
}
}catch (Exception e){
log.error(e.getMessage(),e);
}
@ -342,8 +479,8 @@ public class BookServiceImpl implements BookService {
}
@Override
public void addVisitCount(Long bookId, Integer visitCount) {
bookMapper.addVisitCount(bookId,visitCount);
public void addVisitCount(Long bookId) {
bookMapper.addVisitCount(bookId, new Date());
}
@Override
@ -463,14 +600,14 @@ public class BookServiceImpl implements BookService {
}
@Override
public List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit) {
return bookMapper.queryNetworkPicBooks(localPicPrefix,limit);
public List<Book> queryNetworkPicBooks(Integer limit) {
return bookMapper.queryNetworkPicBooks(limit);
}
@Override
public void updateBookPicToLocal(String picUrl, Long bookId) {
picUrl = fileService.transFile(picUrl, picSavePath);
picUrl = FileUtil.network2Local(picUrl, picSavePath, Constants.LOCAL_PIC_PREFIX);
bookMapper.update(update(book)
.set(BookDynamicSqlSupport.picUrl)

View File

@ -1,60 +0,0 @@
package com.java2nb.novel.service.impl;
import com.github.tobato.fastdfs.domain.StorePath;
import com.github.tobato.fastdfs.service.FastFileStorageClient;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.service.FileService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileInputStream;
/**
* @author 11797
*/
@Service
@RequiredArgsConstructor
@Slf4j
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "fastDfs")
public class FastDfsFileServiceImpl implements FileService {
private final FastFileStorageClient storageClient;
@Value("${fdfs.webUrl}")
private String webUrl;
@Override
public String transFile(String picSrc, String picSavePath) {
File file;
String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
file = new File(picSavePath + filePath);
} else {
//默认图片不存储
return filePath;
}
try {
FileInputStream inputStream = new FileInputStream(file);
StorePath storePath = storageClient.uploadFile(inputStream, file.length(),
FilenameUtils.getExtension(file.getName()), null);
//这里额外加上LOCAL_PIC_PREFIX路径表明该图片是个人资源而不是爬虫爬取的网络资源不需要再次进行转换
// 实际访问时再通过nginx的rewite指令来重写路径去掉LOCAL_PIC_PREFIX
return webUrl+Constants.LOCAL_PIC_PREFIX+storePath.getFullPath();
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
//删除
file.delete();
}
return "/images/default.gif";
}
}

View File

@ -1,23 +0,0 @@
package com.java2nb.novel.service.impl;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.service.FileService;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
/**
* @author 11797
*/
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "local")
public class LocalFileServiceImpl implements FileService {
@Override
public String transFile(String picSrc, String picSavePath){
return FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
}
}

View File

@ -1,76 +0,0 @@
package com.java2nb.novel.service.impl;
import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.CannedAccessControlList;
import com.aliyun.oss.model.CreateBucketRequest;
import com.aliyun.oss.model.PutObjectRequest;
import com.aliyun.oss.model.PutObjectResult;
import com.java2nb.novel.core.config.OssProperties;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.service.FileService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.io.File;
/**
* @author 11797
*/
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "OSS")
@Slf4j
public class OssFileServiceImpl implements FileService {
private final OssProperties ossProperties;
@Override
public String transFile(String picSrc, String picSavePath) {
File file;
String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
file = new File(picSavePath+filePath);
} else {
//默认图片不存储
return filePath;
}
filePath = filePath.replaceFirst(picSavePath,"");
filePath = filePath.startsWith("/") ? filePath.replaceFirst("/","") : filePath;
OSSClient ossClient = new OSSClient(ossProperties.getEndpoint(), ossProperties.getKeyId(), ossProperties.getKeySecret());
try {
//容器不存在,就创建
if (!ossClient.doesBucketExist(ossProperties.getBucketName())) {
ossClient.createBucket(ossProperties.getBucketName());
CreateBucketRequest createBucketRequest = new CreateBucketRequest(ossProperties.getBucketName());
createBucketRequest.setCannedACL(CannedAccessControlList.PublicRead);
ossClient.createBucket(createBucketRequest);
}
//上传文件
PutObjectResult result = ossClient.putObject(new PutObjectRequest(ossProperties.getBucketName(), filePath, file));
//设置权限 这里是公开读
ossClient.setBucketAcl(ossProperties.getBucketName(), CannedAccessControlList.PublicRead);
if(result != null) {
return ossProperties.getWebUrl() + "/" + filePath;
}
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
//关闭
ossClient.shutdown();
file.delete();
}
return "/images/default.gif";
}
}

View File

@ -1,199 +0,0 @@
package com.java2nb.novel.service.impl;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.core.enums.ResponseStatus;
import com.java2nb.novel.core.exception.BusinessException;
import com.java2nb.novel.core.utils.StringUtil;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.search.BookSP;
import com.java2nb.novel.service.SearchService;
import com.java2nb.novel.vo.EsBookVO;
import io.searchbox.client.JestClient;
import io.searchbox.core.*;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @author 11797
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class SearchServiceImpl implements SearchService {
private final String INDEX = "novel";
private final String TYPE = "book";
private final JestClient jestClient;
@Override
@SneakyThrows
public void importToEs(Book book) {
//导入到ES
EsBookVO esBookVO = new EsBookVO();
BeanUtils.copyProperties(book, esBookVO, "lastIndexUpdateTime");
esBookVO.setLastIndexUpdateTime(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(book.getLastIndexUpdateTime()));
Index action = new Index.Builder(esBookVO).index(INDEX).type(TYPE).id(book.getId().toString()).build();
jestClient.execute(action);
}
@SneakyThrows
@Override
public PageInfo searchBook(BookSP params, int page, int pageSize) {
List<EsBookVO> bookList = new ArrayList<>(0);
//使用搜索引擎搜索
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// 构造查询哪个字段
if (StringUtils.isNoneBlank(params.getKeyword())) {
boolQueryBuilder = boolQueryBuilder.must(QueryBuilders.queryStringQuery(params.getKeyword()));
}
// 作品方向
if (params.getWorkDirection() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("workDirection", params.getWorkDirection()));
}
// 分类
if (params.getCatId() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("catId", params.getCatId()));
}
if (params.getBookStatus() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("bookStatus", params.getBookStatus()));
}
if (params.getWordCountMin() == null) {
params.setWordCountMin(0);
}
if (params.getWordCountMax() == null) {
params.setWordCountMax(Integer.MAX_VALUE);
}
boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
if (params.getUpdateTimeMin() != null) {
boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(params.getUpdateTimeMin()));
}
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQueryBuilder);
Count count = new Count.Builder().addIndex(INDEX).addType(TYPE)
.query(searchSourceBuilder.toString()).build();
CountResult results = jestClient.execute(count);
Double total = results.getCount();
// 高亮字段
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.field("authorName");
highlightBuilder.field("bookName");
highlightBuilder.field("bookDesc");
highlightBuilder.field("lastIndexName");
highlightBuilder.field("catName");
highlightBuilder.preTags("<span style='color:red'>").postTags("</span>");
highlightBuilder.fragmentSize(20000);
searchSourceBuilder.highlighter(highlightBuilder);
//设置排序
if (params.getSort() != null) {
searchSourceBuilder.sort(StringUtil.camelName(params.getSort()), SortOrder.DESC);
}
// 设置分页
searchSourceBuilder.from((page - 1) * pageSize);
searchSourceBuilder.size(pageSize);
// 构建Search对象
Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(INDEX).addType(TYPE).build();
log.debug(search.toString());
SearchResult result;
result = jestClient.execute(search);
if (result.isSucceeded()) {
log.debug(result.getJsonString());
Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
if (resultMap.get("hits") != null) {
Map hitsMap = (Map) resultMap.get("hits");
if (hitsMap.size() > 0 && hitsMap.get("hits") != null) {
List hitsList = (List) hitsMap.get("hits");
if (hitsList.size() > 0 && result.getSourceAsString() != null) {
JavaType jt = new ObjectMapper().getTypeFactory().constructParametricType(ArrayList.class, EsBookVO.class);
bookList = new ObjectMapper().readValue("[" + result.getSourceAsString() + "]", jt);
if (bookList != null) {
for (int i = 0; i < bookList.size(); i++) {
hitsMap = (Map) hitsList.get(i);
Map highlightMap = (Map) hitsMap.get("highlight");
if (highlightMap != null && highlightMap.size() > 0) {
List<String> authorNameList = (List<String>) highlightMap.get("authorName");
if (authorNameList != null && authorNameList.size() > 0) {
bookList.get(i).setAuthorName(authorNameList.get(0));
}
List<String> bookNameList = (List<String>) highlightMap.get("bookName");
if (bookNameList != null && bookNameList.size() > 0) {
bookList.get(i).setBookName(bookNameList.get(0));
}
List<String> bookDescList = (List<String>) highlightMap.get("bookDesc");
if (bookDescList != null && bookDescList.size() > 0) {
bookList.get(i).setBookDesc(bookDescList.get(0));
}
List<String> lastIndexNameList = (List<String>) highlightMap.get("lastIndexName");
if (lastIndexNameList != null && lastIndexNameList.size() > 0) {
bookList.get(i).setLastIndexName(lastIndexNameList.get(0));
}
List<String> catNameList = (List<String>) highlightMap.get("catName");
if (catNameList != null && catNameList.size() > 0) {
bookList.get(i).setCatName(catNameList.get(0));
}
}
}
}
}
}
}
PageInfo<EsBookVO> pageInfo = new PageInfo<>(bookList);
pageInfo.setTotal(total.longValue());
pageInfo.setPageNum(page);
pageInfo.setPageSize(pageSize);
return pageInfo;
}
throw new BusinessException(ResponseStatus.ES_SEARCH_FAIL);
}
}

View File

@ -1,16 +0,0 @@
#FastDfs的配置 ====================================
#读取inputsream阻塞时间
fdfs:
connect-timeout: 600
so-timeout: 1500
#tracker地址
trackerList: 127.0.0.1:22122
#缩略图配置
thumbImage:
height: 150
width: 150
#通过nginx 访问地址
webUrl: http://127.0.0.1/
#获取连接池最大数量
pool.max-total: 200

View File

@ -1,9 +0,0 @@
#OSS
novel:
file:
endpoint: oss-cn-shenzhen.aliyuncs.com #不同的服务器地址不同
key-id: dhjjrue6767778878 #去OSS控制台获取
key-secret: dssdkkrkelrkellk44554 #去OSS控制台获取
bucket-name: novel #这个自己创建bucket时的命名控制台创建也行代码创建也行
file-host: pic #文件路径
web-url: https://xxyimg.oss-cn-hangzhou.aliyuncs.com #OSS文件的web访问地址

View File

@ -4,19 +4,7 @@ server:
spring:
profiles:
active: dev
include: alipay,oss,fastdfs
rabbitmq:
enable: 0
host: 127.0.0.1
username: guest
password: guest
virtual-host: /novel-plus
template:
# 缺省的交换机名称此处配置后发送消息如果不指定交换机就会使用这个
exchange: novel.exchange
publisher-confirms: false
include: alipay
elasticsearch:
@ -25,9 +13,7 @@ spring:
jest:
uris: http://127.0.0.1:9200
redisson:
singleServerConfig:
address: 127.0.0.1:6379
jwt:
secret: novel!#20191230

View File

@ -35,7 +35,7 @@
</select>
<update id="addVisitCount" >
update book set visit_count = visit_count + ${visitCount}
update book set visit_count = visit_count + 1 , update_time = #{date}
where id = #{bookId}
</update>
@ -56,7 +56,6 @@
select
id,pic_url from book
where pic_url like 'http%'
and pic_url not like concat('%',#{localPicPrefix},'%')
limit #{limit}
</select>

View File

@ -5,7 +5,7 @@
<groupId>com.java2nb</groupId>
<artifactId>novel</artifactId>
<version>2.5.1</version>
<version>2.2.0</version>
<modules>
<module>novel-common</module>
<module>novel-front</module>
@ -39,12 +39,6 @@
<elasticsearch.version>6.2.2</elasticsearch.version>
<jest.version>6.3.1</jest.version>
<redis.version>1.4.1.RELEASE</redis.version>
<redisson.version>3.12.5</redisson.version>
<sharding.jdbc.version>3.0.0</sharding.jdbc.version>
<aliyun-sdk-oss.version>2.4.0</aliyun-sdk-oss.version>
<commons-fileupload.version>1.3.1</commons-fileupload.version>
<fastdfs-client.version>1.26.1-RELEASE</fastdfs-client.version>
<alipay-sdk-java.version>4.9.153.ALL</alipay-sdk-java.version>
</properties>
<dependencyManagement>

View File

@ -1,2 +0,0 @@
INSERT INTO `crawl_source` (`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES
(6, '新笔趣阁', '{\n \"bookListUrl\": \"http://www.xbiquge.la/fenlei/{catId}_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"6\",\n \"catId6\": \"5\"\n },\n \"bookIdPatten\": \"<a\\\\s+href=\\\"http://www.xbiquge.la/(\\\\d+/\\\\d+)/\\\"\\\\s+target=\\\"_blank\\\">\",\n \"pagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">(\\\\d+)/\\\\d+</em>\",\n \"totalPagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">\\\\d+/(\\\\d+)</em>\",\n \"bookDetailUrl\": \"http://www.xbiquge.la/{bookId}/\",\n \"bookNamePatten\": \"<h1>([^/]+)</h1>\",\n \"authorNamePatten\": \"者:([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.xbiquge.la/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"bookStatusRule\": {},\n \"descStart\": \"<div id=\\\"intro\\\">\",\n \"descEnd\": \"</div>\",\n \"upadateTimePatten\": \"<p>最后更新:(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)</p>\",\n \"upadateTimeFormatPatten\": \"yyyy-MM-dd HH:mm:ss\",\n \"bookIndexUrl\": \"http://www.xbiquge.la/{bookId}/\",\n \"indexIdPatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/(\\\\d+)\\\\.html\'\\\\s+>[^/]+</a>\",\n \"indexNamePatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/\\\\d+\\\\.html\'\\\\s+>([^/]+)</a>\",\n \"bookContentUrl\": \"http://www.xbiquge.la/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\">\",\n \"contentEnd\": \"<p>\"\n}', 0, '2020-05-23 22:46:58', '2020-05-23 22:46:58');

View File

@ -1691,122 +1691,4 @@ INSERT INTO `user_read_history` VALUES ('113', '1255060328322027520', '125467697
INSERT INTO `user_read_history` VALUES ('117', '1255060328322027520', '1254946661743603712', '1254946914001629184', '2020-04-30 19:37:09', '2020-04-30 19:37:09');
INSERT INTO `user_read_history` VALUES ('118', '1255060328322027520', '1254957312633352192', '3335449', '2020-04-30 19:37:36', '2020-04-30 19:37:36');
DROP TABLE IF EXISTS `book_content0`;
CREATE TABLE `book_content0` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1155 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content1
-- ----------------------------
DROP TABLE IF EXISTS `book_content1`;
CREATE TABLE `book_content1` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=406 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content2
-- ----------------------------
DROP TABLE IF EXISTS `book_content2`;
CREATE TABLE `book_content2` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1222 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content3
-- ----------------------------
DROP TABLE IF EXISTS `book_content3`;
CREATE TABLE `book_content3` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=410 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content4
-- ----------------------------
DROP TABLE IF EXISTS `book_content4`;
CREATE TABLE `book_content4` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1188 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content5
-- ----------------------------
DROP TABLE IF EXISTS `book_content5`;
CREATE TABLE `book_content5` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=416 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content6
-- ----------------------------
DROP TABLE IF EXISTS `book_content6`;
CREATE TABLE `book_content6` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1180 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content7
-- ----------------------------
DROP TABLE IF EXISTS `book_content7`;
CREATE TABLE `book_content7` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=404 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content8
-- ----------------------------
DROP TABLE IF EXISTS `book_content8`;
CREATE TABLE `book_content8` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1134 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content9
-- ----------------------------
DROP TABLE IF EXISTS `book_content9`;
CREATE TABLE `book_content9` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n \"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"7\",\n \"catId6\": \"6\",\n \"catId7\": \"8\"\n },\n \"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n \"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n \"totalPagePatten\": \"\\\\d+/(\\\\d+)页\",\n \"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n \"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n \"authorNamePatten\": \"<p>作者:([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"statusPatten\": \"<p>状态:([^/]+)</p>\",\n \"bookStatusRule\": {\n \"连载中\": 0,\n \"完本\": 1\n },\n \"descStart\": \"<div class=\\\"intro_info\\\">\",\n \"descEnd\": \"最新章节推荐地址\",\n \"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n \"bookIndexStart\": \"<dt>《\",\n \"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n \"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n \"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n \"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;