mirror of
https://github.com/201206030/novel-plus.git
synced 2025-07-01 15:26:37 +00:00
Compare commits
1 Commits
release_v2
...
release_v2
Author | SHA1 | Date | |
---|---|---|---|
b1db37bb3d |
36
README.md
36
README.md
@ -1,15 +1,7 @@
|
||||
[]( https://www.eve.bz/cart.php?gid=11 )
|
||||
[](https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console)
|
||||
|
||||
# 小说精品屋-plus
|
||||
|
||||
#### 新项目:小说精品屋-微服务版
|
||||
|
||||
基于小说精品屋-plus构建的Spring Cloud 微服务小说门户平台,可用于学习和商用。
|
||||
|
||||
Gitee仓库地址: https://gitee.com/xiongxyang/novel-cloud
|
||||
|
||||
GitHub仓库地址: https://github.com/201206030/novel-cloud
|
||||
|
||||
#### 演示地址
|
||||
|
||||
[点击前往](http://www.java2nb.com)(前台门户)
|
||||
@ -48,31 +40,7 @@ novel-plus -- 父工程
|
||||
```
|
||||
|
||||
#### 技术选型
|
||||
|
||||
| 技术 | 说明
|
||||
| -------------------- | ---------------------------
|
||||
| SpringBoot | Spring应用快速开发脚手架
|
||||
| MyBatis | 持久层ORM框架
|
||||
| MyBatis Dynamic SQL | Mybatis动态sql
|
||||
| PageHelper | MyBatis分页插件
|
||||
| MyBatisGenerator | 持久层代码生成插件
|
||||
| Sharding-Jdbc | 代码层分库分表中间件
|
||||
| JJWT | JWT登录支持
|
||||
| SpringSecurity | 安全框架
|
||||
| Shiro | 安全框架
|
||||
| Ehcache | Java进程内缓存框架(默认缓存)
|
||||
| Redis | 分布式缓存(缓存替换方案,默认关闭,一行配置开启)
|
||||
| ElasticSearch | 搜索引擎(搜索增强方案,默认关闭,一行配置开启)
|
||||
| RabbitMq | 消息队列(流量削峰,默认关闭,一行配置开启)
|
||||
| OSS | 阿里云对象存储服务(图片存储方式之一,一行配置即可切换)
|
||||
| FastDfs |开源轻量级分布式文件系统(图片存储方式之一,一行配置即可切换)
|
||||
| Redisson | 实现分布式锁
|
||||
| Lombok | 简化对象封装工具
|
||||
| Docker | 应用容器引擎
|
||||
| Mysql | 数据库服务
|
||||
| Thymeleaf | 模板引擎
|
||||
| Layui | 前端UI
|
||||
|
||||
Springboot+Mybatis+Mysql+ElasticSearch+Ehcache+Thymeleaf+Layui
|
||||
|
||||
#### PC站截图
|
||||
|
||||
|
@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>novel</artifactId>
|
||||
<groupId>com.java2nb</groupId>
|
||||
<version>2.5.1</version>
|
||||
<version>2.2.0</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@ -57,22 +57,6 @@
|
||||
<version>${mybatis.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- 分库分表-->
|
||||
<!-- sharding jdbc依赖 -->
|
||||
|
||||
<dependency>
|
||||
<groupId>io.shardingsphere</groupId>
|
||||
<artifactId>sharding-jdbc-spring-boot-starter</artifactId>
|
||||
<version>${sharding.jdbc.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.shardingsphere</groupId>
|
||||
<artifactId>sharding-jdbc-spring-namespace</artifactId>
|
||||
<version>${sharding.jdbc.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<!--分页助手启动器-->
|
||||
<dependency>
|
||||
<groupId>com.github.pagehelper</groupId>
|
||||
|
@ -55,14 +55,4 @@ public interface CacheKey {
|
||||
* 搜索引擎转换锁
|
||||
* */
|
||||
String ES_TRANS_LOCK = "esTransLock";
|
||||
|
||||
/**
|
||||
* 上一次搜索引擎是否更新过小说点击量
|
||||
* */
|
||||
String ES_IS_UPDATE_VISIT = "esIsUpdateVisit";
|
||||
|
||||
/**
|
||||
* 累积的小说点击量
|
||||
* */
|
||||
String BOOK_ADD_VISIT_COUNT = "bookAddVisitCount";
|
||||
}
|
@ -58,16 +58,13 @@ public enum ResponseStatus {
|
||||
, BOOKNAME_EXISTS(4003,"已发布过同名小说!")
|
||||
|
||||
,
|
||||
/**
|
||||
* 搜索引擎相关错误
|
||||
* */
|
||||
ES_SEARCH_FAIL(9001,"密码错误!"),
|
||||
|
||||
|
||||
/**
|
||||
* 其他通用错误
|
||||
* */
|
||||
PASSWORD_ERROR(88001,"密码错误!");
|
||||
|
||||
private int code;
|
||||
private String msg;
|
||||
|
||||
|
@ -24,7 +24,7 @@ public final class BookContentDynamicSqlSupport {
|
||||
|
||||
public final SqlColumn<Long> indexId = column("index_id", JDBCType.BIGINT);
|
||||
|
||||
public final SqlColumn<String> content = column("content", JDBCType.VARCHAR);
|
||||
public final SqlColumn<String> content = column("content", JDBCType.LONGVARCHAR);
|
||||
|
||||
public BookContent() {
|
||||
super("book_content");
|
||||
|
@ -63,7 +63,7 @@ public interface BookContentMapper {
|
||||
@Results(id="BookContentResult", value = {
|
||||
@Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
|
||||
@Result(column="index_id", property="indexId", jdbcType=JdbcType.BIGINT),
|
||||
@Result(column="content", property="content", jdbcType=JdbcType.VARCHAR)
|
||||
@Result(column="content", property="content", jdbcType=JdbcType.LONGVARCHAR)
|
||||
})
|
||||
List<BookContent> selectMany(SelectStatementProvider selectStatement);
|
||||
|
||||
|
@ -1,8 +1,11 @@
|
||||
spring:
|
||||
profiles:
|
||||
include: [common]
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
datasource:
|
||||
url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
|
||||
username: root
|
||||
password: test123456
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
#Redis服务器IP
|
||||
redis:
|
||||
host: 127.0.0.1
|
||||
@ -22,48 +25,6 @@ spring:
|
||||
min-idle: 0
|
||||
#连接超时时间(毫秒)
|
||||
timeout: 30000
|
||||
datasource:
|
||||
url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
|
||||
username: root
|
||||
password: test123456
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
|
||||
####使用shardingJdbc时,
|
||||
####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误
|
||||
##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR
|
||||
|
||||
sharding:
|
||||
jdbc:
|
||||
datasource:
|
||||
names: ds0 #,ds1
|
||||
ds0:
|
||||
type: com.zaxxer.hikari.HikariDataSource
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
|
||||
username: root
|
||||
password: test123456
|
||||
# ds1:
|
||||
# type: com.alibaba.druid.pool.DruidDataSource
|
||||
# driver-class-name: com.mysql.jdbc.Driver
|
||||
# url: jdbc:mysql://localhost:3306/novel_plus2
|
||||
# username: root
|
||||
# password: test123456
|
||||
config:
|
||||
sharding:
|
||||
props:
|
||||
sql.show: true
|
||||
tables:
|
||||
book_content: #book_content表
|
||||
key-generator-column-name: id #主键
|
||||
actual-data-nodes: ds${0}.book_content${0..9} #数据节点
|
||||
# database-strategy: #分库策略
|
||||
# inline:
|
||||
# sharding-column: book_id
|
||||
# algorithm-expression: ds${book_id % 10}
|
||||
table-strategy: #分表策略
|
||||
inline:
|
||||
shardingColumn: index_id
|
||||
algorithm-expression: book_content${index_id % 10}
|
||||
|
||||
|
||||
|
||||
@ -72,8 +33,7 @@ sharding:
|
||||
|
||||
pic:
|
||||
save:
|
||||
type: 2 #图片保存方式, 1不保存,使用爬取的网络图片 ,2保存在自己的存储介质
|
||||
storage: local #存储介质,local:本地,OSS:阿里云对象存储,fastDfs:分布式文件系统
|
||||
type: 2 #图片保存方式, 1不保存,使用网络图片 ,2本地保存
|
||||
path: /var/pic #图片保存路径
|
||||
|
||||
|
||||
|
@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>novel</artifactId>
|
||||
<groupId>com.java2nb</groupId>
|
||||
<version>2.5.1</version>
|
||||
<version>2.2.0</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@ -25,8 +25,6 @@
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-security</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
@ -96,12 +96,8 @@ public class CrawlParser {
|
||||
|
||||
String desc = bookDetailHtml.substring(bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length());
|
||||
desc = desc.substring(0, desc.indexOf(ruleBean.getDescEnd()));
|
||||
//过滤掉简介中的特殊标签
|
||||
desc = desc.replaceAll("<a[^<]+</a>","")
|
||||
.replaceAll("<font[^<]+</font>","")
|
||||
.replaceAll("<p>\\s*</p>","")
|
||||
.replaceAll("<p>","")
|
||||
.replaceAll("</p>","<br/>");
|
||||
//过滤掉简介中的a标签
|
||||
desc = desc.replaceAll("<a[^<]+</a>","");
|
||||
//设置书籍简介
|
||||
book.setBookDesc(desc);
|
||||
if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {
|
||||
|
@ -61,7 +61,7 @@ public class StarterListener implements ServletContextListener {
|
||||
Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
|
||||
//这里只做老书更新
|
||||
book.setId(needUpdateBook.getId());
|
||||
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
|
||||
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
|
||||
//本地图片则不更新
|
||||
book.setPicUrl(null);
|
||||
}
|
||||
|
@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>novel</artifactId>
|
||||
<groupId>com.java2nb</groupId>
|
||||
<version>2.5.1</version>
|
||||
<version>2.2.0</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@ -27,11 +27,6 @@
|
||||
<version>${jjwt.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-amqp</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>io.searchbox</groupId>
|
||||
@ -39,45 +34,16 @@
|
||||
<version>${jest.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!--aliyunOSS-->
|
||||
<dependency>
|
||||
<groupId>com.aliyun.oss</groupId>
|
||||
<artifactId>aliyun-sdk-oss</artifactId>
|
||||
<version>${aliyun-sdk-oss.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-fileupload</groupId>
|
||||
<artifactId>commons-fileupload</artifactId>
|
||||
<version>${commons-fileupload.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!--FastDfs-->
|
||||
<dependency>
|
||||
<groupId>com.github.tobato</groupId>
|
||||
<artifactId>fastdfs-client</artifactId>
|
||||
<version>${fastdfs-client.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
|
||||
<!--引入redisson分布式锁-->
|
||||
<!-- <dependency>
|
||||
<groupId>org.redisson</groupId>
|
||||
<artifactId>redisson-spring-boot-starter</artifactId>
|
||||
<version>${redisson.version}</version>
|
||||
</dependency>-->
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>com.alipay.sdk</groupId>
|
||||
<artifactId>alipay-sdk-java</artifactId>
|
||||
<version>${alipay-sdk-java.version}</version>
|
||||
<version>4.9.153.ALL</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
@ -1,13 +1,10 @@
|
||||
package com.java2nb.novel;
|
||||
|
||||
import com.github.tobato.fastdfs.FdfsClientConfig;
|
||||
import io.shardingsphere.shardingjdbc.spring.boot.SpringBootConfiguration;
|
||||
import org.mybatis.spring.annotation.MapperScan;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.scheduling.TaskScheduler;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
|
||||
@ -21,7 +18,6 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
|
||||
@EnableScheduling
|
||||
@EnableCaching
|
||||
@MapperScan(basePackages = {"com.java2nb.novel.mapper"})
|
||||
@Import(FdfsClientConfig.class)
|
||||
public class FrontNovelApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
@ -4,14 +4,14 @@ import com.github.pagehelper.PageInfo;
|
||||
import com.java2nb.novel.core.bean.ResultBean;
|
||||
import com.java2nb.novel.core.bean.UserDetails;
|
||||
import com.java2nb.novel.core.enums.ResponseStatus;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.entity.BookComment;
|
||||
import com.java2nb.novel.entity.BookIndex;
|
||||
import com.java2nb.novel.search.BookSP;
|
||||
import com.java2nb.novel.service.BookService;
|
||||
import com.java2nb.novel.vo.BookVO;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
@ -19,6 +19,7 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
@ -32,11 +33,6 @@ public class BookController extends BaseController{
|
||||
|
||||
private final BookService bookService;
|
||||
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Value("${spring.rabbitmq.enable}")
|
||||
private Integer enableMq;
|
||||
|
||||
|
||||
/**
|
||||
* 查询首页小说设置列表数据
|
||||
@ -109,11 +105,7 @@ public class BookController extends BaseController{
|
||||
* */
|
||||
@PostMapping("addVisitCount")
|
||||
public ResultBean addVisitCount(Long bookId){
|
||||
if(enableMq == 1) {
|
||||
rabbitTemplate.convertAndSend("ADD-BOOK-VISIT-EXCHANGE", null, bookId);
|
||||
}else {
|
||||
bookService.addVisitCount(bookId, 1);
|
||||
}
|
||||
bookService.addVisitCount(bookId);
|
||||
return ResultBean.ok();
|
||||
}
|
||||
|
||||
|
@ -1,13 +0,0 @@
|
||||
package com.java2nb.novel.core.config;
|
||||
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.EnableMBeanExport;
|
||||
import org.springframework.jmx.support.RegistrationPolicy;
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
@Configuration
|
||||
@EnableMBeanExport(registration= RegistrationPolicy.IGNORE_EXISTING)
|
||||
public class FdfsConfiguration {
|
||||
}
|
@ -1,30 +0,0 @@
|
||||
package com.java2nb.novel.core.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
@Data
|
||||
@Component
|
||||
@ConfigurationProperties(prefix="novel.file")
|
||||
public class OssProperties{
|
||||
|
||||
private String endpoint;
|
||||
|
||||
private String keyId;
|
||||
|
||||
private String keySecret;
|
||||
|
||||
private String fileHost;
|
||||
|
||||
private String bucketName;
|
||||
|
||||
private String webUrl;
|
||||
|
||||
|
||||
}
|
@ -1,59 +0,0 @@
|
||||
package com.java2nb.novel.core.config;
|
||||
|
||||
import org.springframework.amqp.core.*;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
@Configuration
|
||||
@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "enable", havingValue = "1")
|
||||
public class RabbitConfig {
|
||||
|
||||
|
||||
/**
|
||||
* 更新数据库队列
|
||||
*/
|
||||
@Bean
|
||||
public Queue updateDbQueue() {
|
||||
return new Queue("UPDATE-DB-QUEUE", true);
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新数据库队列
|
||||
*/
|
||||
@Bean
|
||||
public Queue updateEsQueue() {
|
||||
return new Queue("UPDATE-ES-QUEUE", true);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 增加点击量交换机
|
||||
*/
|
||||
@Bean
|
||||
public FanoutExchange addVisitExchange() {
|
||||
return new FanoutExchange("ADD-BOOK-VISIT-EXCHANGE");
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新搜索引擎队列绑定到增加点击量交换机中
|
||||
*/
|
||||
@Bean
|
||||
public Binding updateEsBinding() {
|
||||
|
||||
return BindingBuilder.bind(updateEsQueue()).to(addVisitExchange());
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新数据库绑定到增加点击量交换机中
|
||||
*/
|
||||
@Bean
|
||||
public Binding updateDbBinding() {
|
||||
return BindingBuilder.bind(updateDbQueue()).to(addVisitExchange());
|
||||
}
|
||||
|
||||
|
||||
}
|
@ -1,107 +0,0 @@
|
||||
package com.java2nb.novel.core.listener;
|
||||
|
||||
import com.java2nb.novel.core.cache.CacheKey;
|
||||
import com.java2nb.novel.core.cache.CacheService;
|
||||
import com.java2nb.novel.core.utils.Constants;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.service.BookService;
|
||||
import com.java2nb.novel.service.SearchService;
|
||||
import com.java2nb.novel.vo.EsBookVO;
|
||||
import com.rabbitmq.client.Channel;
|
||||
import io.searchbox.client.JestClient;
|
||||
import io.searchbox.core.Index;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "enable", havingValue = "1")
|
||||
public class BookVisitAddListener {
|
||||
|
||||
private final BookService bookService;
|
||||
|
||||
private final CacheService cacheService;
|
||||
|
||||
private final SearchService searchService;
|
||||
|
||||
// private final RedissonClient redissonClient;
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 更新数据库
|
||||
* 流量削峰,每本小说累积10个点击更新一次
|
||||
*/
|
||||
@SneakyThrows
|
||||
@RabbitListener(queues = {"UPDATE-DB-QUEUE"})
|
||||
public void updateDb(Long bookId, Channel channel, Message message) {
|
||||
|
||||
log.debug("收到更新数据库消息:" + bookId);
|
||||
Thread.sleep(1000 * 2);
|
||||
//TODO 操作共享资源visitCount,集群环境下有线程安全问题,引入Redisson框架实现分布式锁
|
||||
//RLock lock = redissonClient.getLock("visitCount");
|
||||
//lock.lock();
|
||||
|
||||
|
||||
//目前visitCount不重要,数据可丢失,暂不实现分布式锁
|
||||
Integer visitCount = (Integer) cacheService.getObject(CacheKey.BOOK_ADD_VISIT_COUNT+bookId);
|
||||
if(visitCount == null){
|
||||
visitCount = 0 ;
|
||||
}
|
||||
cacheService.setObject(CacheKey.BOOK_ADD_VISIT_COUNT+bookId,++visitCount);
|
||||
if(visitCount >= Constants.ADD_MAX_VISIT_COUNT) {
|
||||
bookService.addVisitCount(bookId,visitCount);
|
||||
cacheService.del(CacheKey.BOOK_ADD_VISIT_COUNT+bookId);
|
||||
}
|
||||
|
||||
//TODO 操作共享资源visitCount,集群环境下有线程安全问题,引入Redisson框架实现分布式锁
|
||||
//lock.unlock();
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新搜索引擎
|
||||
* 流量削峰,每本小说1个小时更新一次
|
||||
*/
|
||||
@RabbitListener(queues = {"UPDATE-ES-QUEUE"})
|
||||
public void updateEs(Long bookId, Channel channel, Message message) {
|
||||
|
||||
log.debug("收到更新搜索引擎消息:" + bookId);
|
||||
if (cacheService.get(CacheKey.ES_IS_UPDATE_VISIT + bookId) == null) {
|
||||
cacheService.set(CacheKey.ES_IS_UPDATE_VISIT + bookId, "1", 60 * 60);
|
||||
try {
|
||||
Thread.sleep(1000 * 5);
|
||||
Book book = bookService.queryBookDetail(bookId);
|
||||
searchService.importToEs(book);
|
||||
}catch (Exception e){
|
||||
cacheService.del(CacheKey.ES_IS_UPDATE_VISIT + bookId);
|
||||
log.error("更新搜索引擎失败"+bookId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
@ -5,7 +5,6 @@ import com.java2nb.novel.core.cache.CacheService;
|
||||
import com.java2nb.novel.core.utils.BeanUtil;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.service.BookService;
|
||||
import com.java2nb.novel.service.SearchService;
|
||||
import com.java2nb.novel.vo.EsBookVO;
|
||||
import io.searchbox.client.JestClient;
|
||||
import io.searchbox.core.DocumentResult;
|
||||
@ -38,20 +37,16 @@ public class BookToEsSchedule {
|
||||
|
||||
private final CacheService cacheService;
|
||||
|
||||
|
||||
|
||||
private final SearchService searchService;
|
||||
private final JestClient jestClient;
|
||||
|
||||
|
||||
/**
|
||||
* 1分钟导入一次
|
||||
* 10秒钟导入一次
|
||||
*/
|
||||
@Scheduled(fixedRate = 1000 * 60)
|
||||
@Scheduled(fixedRate = 1000 * 10)
|
||||
public void saveToEs() {
|
||||
//TODO 引入Redisson框架实现分布式锁
|
||||
//可以重复更新,只是效率可能略有降低,所以暂不实现分布式锁
|
||||
if (cacheService.get(CacheKey.ES_TRANS_LOCK) == null) {
|
||||
cacheService.set(CacheKey.ES_TRANS_LOCK, "1", 60 * 20);
|
||||
cacheService.set(CacheKey.ES_TRANS_LOCK, "1", 60 * 5);
|
||||
try {
|
||||
//查询需要更新的小说
|
||||
Date lastDate = (Date) cacheService.getObject(CacheKey.ES_LAST_UPDATE_TIME);
|
||||
@ -62,9 +57,17 @@ public class BookToEsSchedule {
|
||||
|
||||
List<Book> books = bookService.queryBookByUpdateTimeByPage(lastDate, 100);
|
||||
for (Book book : books) {
|
||||
searchService.importToEs(book);
|
||||
//导入到ES
|
||||
EsBookVO esBookVO = new EsBookVO();
|
||||
BeanUtils.copyProperties(book, esBookVO, "lastIndexUpdateTime");
|
||||
esBookVO.setLastIndexUpdateTime(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(book.getLastIndexUpdateTime()));
|
||||
Index action = new Index.Builder(esBookVO).index("novel").type("book").id(book.getId().toString()).build();
|
||||
|
||||
jestClient.execute(action);
|
||||
|
||||
lastDate = book.getUpdateTime();
|
||||
Thread.sleep(5000);
|
||||
|
||||
Thread.sleep(1000);
|
||||
|
||||
}
|
||||
|
||||
|
@ -1,6 +1,5 @@
|
||||
package com.java2nb.novel.core.schedule;
|
||||
|
||||
import com.java2nb.novel.core.utils.Constants;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.service.BookService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 将爬取的网络图片转存为自己的存储介质(本地、OSS、fastDfs)任务
|
||||
* 网络图片转存本地任务
|
||||
*
|
||||
* @author Administrator
|
||||
*/
|
||||
@ -22,7 +21,7 @@ import java.util.List;
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CrawlPicTransSchedule {
|
||||
public class Network2LocalPicSchedule {
|
||||
|
||||
private final BookService bookService;
|
||||
|
||||
@ -42,7 +41,7 @@ public class CrawlPicTransSchedule {
|
||||
log.info("Network2LocalPicSchedule。。。。。。。。。。。。");
|
||||
|
||||
|
||||
List<Book> networkPicBooks = bookService.queryNetworkPicBooks(Constants.LOCAL_PIC_PREFIX,100);
|
||||
List<Book> networkPicBooks = bookService.queryNetworkPicBooks(100);
|
||||
for (Book book : networkPicBooks) {
|
||||
bookService.updateBookPicToLocal(book.getPicUrl(), book.getId());
|
||||
//3秒钟转化一张图片,10分钟转化200张
|
@ -12,7 +12,7 @@ public class Constants {
|
||||
public static final String TEMPLATE_PATH_PREFIX_KEY = "templatePathPrefixKey";
|
||||
|
||||
/**
|
||||
* 保存图片到自己的存储介质路径前缀
|
||||
* 本地图片保存前缀
|
||||
* */
|
||||
public static final String LOCAL_PIC_PREFIX = "/localPic/";
|
||||
|
||||
@ -30,9 +30,4 @@ public class Constants {
|
||||
* 首页设置的小说数量
|
||||
* */
|
||||
public static final int INDEX_BOOK_SETTING_NUM = 32;
|
||||
|
||||
/**
|
||||
* 累积的最大点击量
|
||||
* */
|
||||
public static final Integer ADD_MAX_VISIT_COUNT = 10;
|
||||
}
|
||||
|
@ -16,13 +16,13 @@ public interface FrontBookMapper extends BookMapper {
|
||||
|
||||
List<BookVO> searchByPage(BookSP params);
|
||||
|
||||
void addVisitCount(@Param("bookId") Long bookId, @Param("visitCount") Integer visitCount);
|
||||
void addVisitCount(@Param("bookId") Long bookId, @Param("date") Date date);
|
||||
|
||||
List<Book> listRecBookByCatId(@Param("catId") Integer catId);
|
||||
|
||||
void addCommentCount(@Param("bookId") Long bookId);
|
||||
|
||||
List<Book> queryNetworkPicBooks(@Param("localPicPrefix") String localPicPrefix, @Param("limit") Integer limit);
|
||||
List<Book> queryNetworkPicBooks(@Param("limit") Integer limit);
|
||||
|
||||
/**
|
||||
* 按评分随机查询小说集合
|
||||
|
@ -112,8 +112,8 @@ public interface BookService {
|
||||
/**
|
||||
* 增加点击次数
|
||||
* @param bookId 书籍ID
|
||||
* @param visitCount*/
|
||||
void addVisitCount(Long bookId, Integer visitCount);
|
||||
* */
|
||||
void addVisitCount(Long bookId);
|
||||
|
||||
/**
|
||||
* 查询章节数
|
||||
@ -180,17 +180,15 @@ public interface BookService {
|
||||
|
||||
/**
|
||||
* 查询网络图片的小说
|
||||
*
|
||||
* @param localPicPrefix
|
||||
* @param limit 查询条数
|
||||
* @return 返回小说集合
|
||||
* */
|
||||
List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit);
|
||||
List<Book> queryNetworkPicBooks(Integer limit);
|
||||
|
||||
|
||||
/**
|
||||
* 更新爬取的小说网络图片到自己的存储介质(本地、OSS、fastDfs)
|
||||
* @param picUrl 爬取的网络图片路径
|
||||
* 更新小说网络图片到本地
|
||||
* @param picUrl 网络图片路径
|
||||
* @param bookId 小说ID
|
||||
*/
|
||||
void updateBookPicToLocal(String picUrl, Long bookId);
|
||||
@ -200,7 +198,6 @@ public interface BookService {
|
||||
* @param userId 用户ID
|
||||
* @param page 页码
|
||||
* @param pageSize 分页大小
|
||||
* @return 小说集合
|
||||
* */
|
||||
List<Book> listBookPageByUserId(Long userId, int page, int pageSize);
|
||||
|
||||
|
@ -1,17 +0,0 @@
|
||||
package com.java2nb.novel.service;
|
||||
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
public interface FileService {
|
||||
|
||||
/**
|
||||
* 将爬取的网络图片转存为自己的存储介质(本地、OSS、fastDfs)
|
||||
* @param picSrc 爬取的网络图片路径
|
||||
* @param picSavePath 保存路径
|
||||
* @return 新图片地址
|
||||
* */
|
||||
String transFile(String picSrc, String picSavePath);
|
||||
|
||||
}
|
@ -1,27 +0,0 @@
|
||||
package com.java2nb.novel.service;
|
||||
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.search.BookSP;
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
public interface SearchService {
|
||||
|
||||
/**
|
||||
* 导入到es
|
||||
* @param book 小说数据
|
||||
*/
|
||||
void importToEs(Book book);
|
||||
|
||||
/**
|
||||
* 搜索
|
||||
* @param params 搜索参数
|
||||
* @param page 当前页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 分页信息
|
||||
*/
|
||||
PageInfo searchBook(BookSP params, int page, int pageSize);
|
||||
}
|
@ -1,5 +1,6 @@
|
||||
package com.java2nb.novel.service.impl;
|
||||
|
||||
import com.fasterxml.jackson.databind.JavaType;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.github.pagehelper.PageHelper;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
@ -14,15 +15,21 @@ import com.java2nb.novel.mapper.*;
|
||||
import com.java2nb.novel.search.BookSP;
|
||||
import com.java2nb.novel.service.AuthorService;
|
||||
import com.java2nb.novel.service.BookService;
|
||||
import com.java2nb.novel.service.FileService;
|
||||
import com.java2nb.novel.service.SearchService;
|
||||
import com.java2nb.novel.vo.BookCommentVO;
|
||||
import com.java2nb.novel.vo.BookSettingVO;
|
||||
import com.java2nb.novel.vo.BookVO;
|
||||
import com.java2nb.novel.vo.EsBookVO;
|
||||
import io.searchbox.client.JestClient;
|
||||
import io.searchbox.core.*;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
import org.mybatis.dynamic.sql.SortSpecification;
|
||||
import org.mybatis.dynamic.sql.render.RenderingStrategies;
|
||||
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
|
||||
@ -32,6 +39,7 @@ import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import tk.mybatis.orderbyhelper.OrderByHelper;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
@ -81,9 +89,7 @@ public class BookServiceImpl implements BookService {
|
||||
|
||||
private final AuthorService authorService;
|
||||
|
||||
private final SearchService searchService;
|
||||
|
||||
private final FileService fileService;
|
||||
private final JestClient jestClient;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@ -194,8 +200,139 @@ public class BookServiceImpl implements BookService {
|
||||
|
||||
|
||||
try {
|
||||
return searchService.searchBook(params,page,pageSize);
|
||||
List<EsBookVO> bookList = new ArrayList<>(0);
|
||||
|
||||
//使用搜索引擎搜索
|
||||
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
|
||||
// 构造查询哪个字段
|
||||
if (StringUtils.isNoneBlank(params.getKeyword())) {
|
||||
boolQueryBuilder = boolQueryBuilder.must(QueryBuilders.queryStringQuery(params.getKeyword()));
|
||||
}
|
||||
|
||||
// 作品方向
|
||||
if (params.getWorkDirection() != null) {
|
||||
boolQueryBuilder.filter(QueryBuilders.termQuery("workDirection", params.getWorkDirection()));
|
||||
}
|
||||
|
||||
// 分类
|
||||
if (params.getCatId() != null) {
|
||||
boolQueryBuilder.filter(QueryBuilders.termQuery("catId", params.getCatId()));
|
||||
}
|
||||
if (params.getBookStatus() != null) {
|
||||
boolQueryBuilder.filter(QueryBuilders.termQuery("bookStatus", params.getBookStatus()));
|
||||
}
|
||||
|
||||
if (params.getWordCountMin() == null) {
|
||||
params.setWordCountMin(0);
|
||||
}
|
||||
if (params.getWordCountMax() == null) {
|
||||
params.setWordCountMax(Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
|
||||
|
||||
if (params.getUpdateTimeMin() != null) {
|
||||
boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(params.getUpdateTimeMin()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(boolQueryBuilder);
|
||||
|
||||
|
||||
Count count = new Count.Builder().addIndex("novel").addType("book")
|
||||
.query(searchSourceBuilder.toString()).build();
|
||||
CountResult results = jestClient.execute(count);
|
||||
Double total = results.getCount();
|
||||
|
||||
|
||||
// 高亮字段
|
||||
HighlightBuilder highlightBuilder = new HighlightBuilder();
|
||||
highlightBuilder.field("authorName");
|
||||
highlightBuilder.field("bookName");
|
||||
highlightBuilder.field("bookDesc");
|
||||
highlightBuilder.field("lastIndexName");
|
||||
highlightBuilder.field("catName");
|
||||
highlightBuilder.preTags("<span style='color:red'>").postTags("</span>");
|
||||
highlightBuilder.fragmentSize(20000);
|
||||
searchSourceBuilder.highlighter(highlightBuilder);
|
||||
|
||||
|
||||
//设置排序
|
||||
if (params.getSort() != null) {
|
||||
searchSourceBuilder.sort(StringUtil.camelName(params.getSort()), SortOrder.DESC);
|
||||
}
|
||||
|
||||
// 设置分页
|
||||
searchSourceBuilder.from((page - 1) * pageSize);
|
||||
searchSourceBuilder.size(pageSize);
|
||||
|
||||
// 构建Search对象
|
||||
Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex("novel").addType("book").build();
|
||||
log.debug(search.toString());
|
||||
SearchResult result;
|
||||
result = jestClient.execute(search);
|
||||
if (result.isSucceeded()) {
|
||||
log.debug(result.getJsonString());
|
||||
|
||||
Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
|
||||
if (resultMap.get("hits") != null) {
|
||||
Map hitsMap = (Map) resultMap.get("hits");
|
||||
if (hitsMap.size() > 0 && hitsMap.get("hits") != null) {
|
||||
List hitsList = (List) hitsMap.get("hits");
|
||||
if (hitsList.size() > 0 && result.getSourceAsString() != null) {
|
||||
|
||||
JavaType jt = new ObjectMapper().getTypeFactory().constructParametricType(ArrayList.class, EsBookVO.class);
|
||||
bookList = new ObjectMapper().readValue("[" + result.getSourceAsString() + "]", jt);
|
||||
|
||||
if (bookList != null) {
|
||||
for (int i = 0; i < bookList.size(); i++) {
|
||||
hitsMap = (Map) hitsList.get(i);
|
||||
Map highlightMap = (Map) hitsMap.get("highlight");
|
||||
if (highlightMap != null && highlightMap.size() > 0) {
|
||||
|
||||
List<String> authorNameList = (List<String>) highlightMap.get("authorName");
|
||||
if (authorNameList != null && authorNameList.size() > 0) {
|
||||
bookList.get(i).setAuthorName(authorNameList.get(0));
|
||||
}
|
||||
|
||||
List<String> bookNameList = (List<String>) highlightMap.get("bookName");
|
||||
if (bookNameList != null && bookNameList.size() > 0) {
|
||||
bookList.get(i).setBookName(bookNameList.get(0));
|
||||
}
|
||||
|
||||
List<String> bookDescList = (List<String>) highlightMap.get("bookDesc");
|
||||
if (bookDescList != null && bookDescList.size() > 0) {
|
||||
bookList.get(i).setBookDesc(bookDescList.get(0));
|
||||
}
|
||||
|
||||
List<String> lastIndexNameList = (List<String>) highlightMap.get("lastIndexName");
|
||||
if (lastIndexNameList != null && lastIndexNameList.size() > 0) {
|
||||
bookList.get(i).setLastIndexName(lastIndexNameList.get(0));
|
||||
}
|
||||
|
||||
List<String> catNameList = (List<String>) highlightMap.get("catName");
|
||||
if (catNameList != null && catNameList.size() > 0) {
|
||||
bookList.get(i).setCatName(catNameList.get(0));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PageInfo<EsBookVO> pageInfo = new PageInfo<>(bookList);
|
||||
pageInfo.setTotal(total.longValue());
|
||||
pageInfo.setPageNum(page);
|
||||
pageInfo.setPageSize(pageSize);
|
||||
return pageInfo;
|
||||
}
|
||||
}catch (Exception e){
|
||||
log.error(e.getMessage(),e);
|
||||
}
|
||||
@ -342,8 +479,8 @@ public class BookServiceImpl implements BookService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addVisitCount(Long bookId, Integer visitCount) {
|
||||
bookMapper.addVisitCount(bookId,visitCount);
|
||||
public void addVisitCount(Long bookId) {
|
||||
bookMapper.addVisitCount(bookId, new Date());
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -463,14 +600,14 @@ public class BookServiceImpl implements BookService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit) {
|
||||
return bookMapper.queryNetworkPicBooks(localPicPrefix,limit);
|
||||
public List<Book> queryNetworkPicBooks(Integer limit) {
|
||||
return bookMapper.queryNetworkPicBooks(limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateBookPicToLocal(String picUrl, Long bookId) {
|
||||
|
||||
picUrl = fileService.transFile(picUrl, picSavePath);
|
||||
picUrl = FileUtil.network2Local(picUrl, picSavePath, Constants.LOCAL_PIC_PREFIX);
|
||||
|
||||
bookMapper.update(update(book)
|
||||
.set(BookDynamicSqlSupport.picUrl)
|
||||
|
@ -1,60 +0,0 @@
|
||||
package com.java2nb.novel.service.impl;
|
||||
|
||||
import com.github.tobato.fastdfs.domain.StorePath;
|
||||
import com.github.tobato.fastdfs.service.FastFileStorageClient;
|
||||
import com.java2nb.novel.core.utils.Constants;
|
||||
import com.java2nb.novel.core.utils.FileUtil;
|
||||
import com.java2nb.novel.service.FileService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "fastDfs")
|
||||
public class FastDfsFileServiceImpl implements FileService {
|
||||
|
||||
private final FastFileStorageClient storageClient;
|
||||
|
||||
@Value("${fdfs.webUrl}")
|
||||
private String webUrl;
|
||||
|
||||
@Override
|
||||
public String transFile(String picSrc, String picSavePath) {
|
||||
|
||||
File file;
|
||||
String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
|
||||
if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
|
||||
file = new File(picSavePath + filePath);
|
||||
} else {
|
||||
//默认图片不存储
|
||||
return filePath;
|
||||
}
|
||||
|
||||
try {
|
||||
FileInputStream inputStream = new FileInputStream(file);
|
||||
StorePath storePath = storageClient.uploadFile(inputStream, file.length(),
|
||||
FilenameUtils.getExtension(file.getName()), null);
|
||||
//这里额外加上LOCAL_PIC_PREFIX路径,表明该图片是个人资源,而不是爬虫爬取的网络资源,不需要再次进行转换,
|
||||
// 实际访问时,再通过nginx的rewite指令来重写路径,去掉LOCAL_PIC_PREFIX
|
||||
return webUrl+Constants.LOCAL_PIC_PREFIX+storePath.getFullPath();
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
} finally {
|
||||
//删除
|
||||
file.delete();
|
||||
}
|
||||
|
||||
return "/images/default.gif";
|
||||
}
|
||||
}
|
@ -1,23 +0,0 @@
|
||||
package com.java2nb.novel.service.impl;
|
||||
|
||||
import com.java2nb.novel.core.utils.Constants;
|
||||
import com.java2nb.novel.core.utils.FileUtil;
|
||||
import com.java2nb.novel.service.FileService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "local")
|
||||
public class LocalFileServiceImpl implements FileService {
|
||||
|
||||
@Override
|
||||
public String transFile(String picSrc, String picSavePath){
|
||||
|
||||
return FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
|
||||
}
|
||||
}
|
@ -1,76 +0,0 @@
|
||||
package com.java2nb.novel.service.impl;
|
||||
|
||||
import com.aliyun.oss.OSSClient;
|
||||
import com.aliyun.oss.model.CannedAccessControlList;
|
||||
import com.aliyun.oss.model.CreateBucketRequest;
|
||||
import com.aliyun.oss.model.PutObjectRequest;
|
||||
import com.aliyun.oss.model.PutObjectResult;
|
||||
import com.java2nb.novel.core.config.OssProperties;
|
||||
import com.java2nb.novel.core.utils.Constants;
|
||||
import com.java2nb.novel.core.utils.FileUtil;
|
||||
import com.java2nb.novel.service.FileService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "OSS")
|
||||
@Slf4j
|
||||
public class OssFileServiceImpl implements FileService {
|
||||
|
||||
private final OssProperties ossProperties;
|
||||
|
||||
@Override
|
||||
public String transFile(String picSrc, String picSavePath) {
|
||||
|
||||
File file;
|
||||
String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
|
||||
if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
|
||||
file = new File(picSavePath+filePath);
|
||||
} else {
|
||||
//默认图片不存储
|
||||
return filePath;
|
||||
}
|
||||
|
||||
filePath = filePath.replaceFirst(picSavePath,"");
|
||||
|
||||
filePath = filePath.startsWith("/") ? filePath.replaceFirst("/","") : filePath;
|
||||
|
||||
|
||||
OSSClient ossClient = new OSSClient(ossProperties.getEndpoint(), ossProperties.getKeyId(), ossProperties.getKeySecret());
|
||||
try {
|
||||
//容器不存在,就创建
|
||||
if (!ossClient.doesBucketExist(ossProperties.getBucketName())) {
|
||||
ossClient.createBucket(ossProperties.getBucketName());
|
||||
CreateBucketRequest createBucketRequest = new CreateBucketRequest(ossProperties.getBucketName());
|
||||
createBucketRequest.setCannedACL(CannedAccessControlList.PublicRead);
|
||||
ossClient.createBucket(createBucketRequest);
|
||||
}
|
||||
//上传文件
|
||||
PutObjectResult result = ossClient.putObject(new PutObjectRequest(ossProperties.getBucketName(), filePath, file));
|
||||
//设置权限 这里是公开读
|
||||
ossClient.setBucketAcl(ossProperties.getBucketName(), CannedAccessControlList.PublicRead);
|
||||
|
||||
if(result != null) {
|
||||
return ossProperties.getWebUrl() + "/" + filePath;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
} finally {
|
||||
//关闭
|
||||
ossClient.shutdown();
|
||||
file.delete();
|
||||
}
|
||||
|
||||
return "/images/default.gif";
|
||||
}
|
||||
|
||||
|
||||
}
|
@ -1,199 +0,0 @@
|
||||
package com.java2nb.novel.service.impl;
|
||||
|
||||
import com.fasterxml.jackson.databind.JavaType;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.java2nb.novel.core.enums.ResponseStatus;
|
||||
import com.java2nb.novel.core.exception.BusinessException;
|
||||
import com.java2nb.novel.core.utils.StringUtil;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.search.BookSP;
|
||||
import com.java2nb.novel.service.SearchService;
|
||||
import com.java2nb.novel.vo.EsBookVO;
|
||||
import io.searchbox.client.JestClient;
|
||||
import io.searchbox.core.*;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author 11797
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class SearchServiceImpl implements SearchService {
|
||||
|
||||
private final String INDEX = "novel";
|
||||
|
||||
private final String TYPE = "book";
|
||||
|
||||
private final JestClient jestClient;
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void importToEs(Book book) {
|
||||
//导入到ES
|
||||
EsBookVO esBookVO = new EsBookVO();
|
||||
BeanUtils.copyProperties(book, esBookVO, "lastIndexUpdateTime");
|
||||
esBookVO.setLastIndexUpdateTime(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(book.getLastIndexUpdateTime()));
|
||||
Index action = new Index.Builder(esBookVO).index(INDEX).type(TYPE).id(book.getId().toString()).build();
|
||||
|
||||
jestClient.execute(action);
|
||||
|
||||
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public PageInfo searchBook(BookSP params, int page, int pageSize) {
|
||||
List<EsBookVO> bookList = new ArrayList<>(0);
|
||||
|
||||
//使用搜索引擎搜索
|
||||
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
|
||||
// 构造查询哪个字段
|
||||
if (StringUtils.isNoneBlank(params.getKeyword())) {
|
||||
boolQueryBuilder = boolQueryBuilder.must(QueryBuilders.queryStringQuery(params.getKeyword()));
|
||||
}
|
||||
|
||||
// 作品方向
|
||||
if (params.getWorkDirection() != null) {
|
||||
boolQueryBuilder.filter(QueryBuilders.termQuery("workDirection", params.getWorkDirection()));
|
||||
}
|
||||
|
||||
// 分类
|
||||
if (params.getCatId() != null) {
|
||||
boolQueryBuilder.filter(QueryBuilders.termQuery("catId", params.getCatId()));
|
||||
}
|
||||
if (params.getBookStatus() != null) {
|
||||
boolQueryBuilder.filter(QueryBuilders.termQuery("bookStatus", params.getBookStatus()));
|
||||
}
|
||||
|
||||
if (params.getWordCountMin() == null) {
|
||||
params.setWordCountMin(0);
|
||||
}
|
||||
if (params.getWordCountMax() == null) {
|
||||
params.setWordCountMax(Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
|
||||
|
||||
if (params.getUpdateTimeMin() != null) {
|
||||
boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(params.getUpdateTimeMin()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(boolQueryBuilder);
|
||||
|
||||
|
||||
Count count = new Count.Builder().addIndex(INDEX).addType(TYPE)
|
||||
.query(searchSourceBuilder.toString()).build();
|
||||
CountResult results = jestClient.execute(count);
|
||||
Double total = results.getCount();
|
||||
|
||||
|
||||
// 高亮字段
|
||||
HighlightBuilder highlightBuilder = new HighlightBuilder();
|
||||
highlightBuilder.field("authorName");
|
||||
highlightBuilder.field("bookName");
|
||||
highlightBuilder.field("bookDesc");
|
||||
highlightBuilder.field("lastIndexName");
|
||||
highlightBuilder.field("catName");
|
||||
highlightBuilder.preTags("<span style='color:red'>").postTags("</span>");
|
||||
highlightBuilder.fragmentSize(20000);
|
||||
searchSourceBuilder.highlighter(highlightBuilder);
|
||||
|
||||
|
||||
//设置排序
|
||||
if (params.getSort() != null) {
|
||||
searchSourceBuilder.sort(StringUtil.camelName(params.getSort()), SortOrder.DESC);
|
||||
}
|
||||
|
||||
// 设置分页
|
||||
searchSourceBuilder.from((page - 1) * pageSize);
|
||||
searchSourceBuilder.size(pageSize);
|
||||
|
||||
// 构建Search对象
|
||||
Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(INDEX).addType(TYPE).build();
|
||||
log.debug(search.toString());
|
||||
SearchResult result;
|
||||
result = jestClient.execute(search);
|
||||
if (result.isSucceeded()) {
|
||||
log.debug(result.getJsonString());
|
||||
|
||||
Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
|
||||
if (resultMap.get("hits") != null) {
|
||||
Map hitsMap = (Map) resultMap.get("hits");
|
||||
if (hitsMap.size() > 0 && hitsMap.get("hits") != null) {
|
||||
List hitsList = (List) hitsMap.get("hits");
|
||||
if (hitsList.size() > 0 && result.getSourceAsString() != null) {
|
||||
|
||||
JavaType jt = new ObjectMapper().getTypeFactory().constructParametricType(ArrayList.class, EsBookVO.class);
|
||||
bookList = new ObjectMapper().readValue("[" + result.getSourceAsString() + "]", jt);
|
||||
|
||||
if (bookList != null) {
|
||||
for (int i = 0; i < bookList.size(); i++) {
|
||||
hitsMap = (Map) hitsList.get(i);
|
||||
Map highlightMap = (Map) hitsMap.get("highlight");
|
||||
if (highlightMap != null && highlightMap.size() > 0) {
|
||||
|
||||
List<String> authorNameList = (List<String>) highlightMap.get("authorName");
|
||||
if (authorNameList != null && authorNameList.size() > 0) {
|
||||
bookList.get(i).setAuthorName(authorNameList.get(0));
|
||||
}
|
||||
|
||||
List<String> bookNameList = (List<String>) highlightMap.get("bookName");
|
||||
if (bookNameList != null && bookNameList.size() > 0) {
|
||||
bookList.get(i).setBookName(bookNameList.get(0));
|
||||
}
|
||||
|
||||
List<String> bookDescList = (List<String>) highlightMap.get("bookDesc");
|
||||
if (bookDescList != null && bookDescList.size() > 0) {
|
||||
bookList.get(i).setBookDesc(bookDescList.get(0));
|
||||
}
|
||||
|
||||
List<String> lastIndexNameList = (List<String>) highlightMap.get("lastIndexName");
|
||||
if (lastIndexNameList != null && lastIndexNameList.size() > 0) {
|
||||
bookList.get(i).setLastIndexName(lastIndexNameList.get(0));
|
||||
}
|
||||
|
||||
List<String> catNameList = (List<String>) highlightMap.get("catName");
|
||||
if (catNameList != null && catNameList.size() > 0) {
|
||||
bookList.get(i).setCatName(catNameList.get(0));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PageInfo<EsBookVO> pageInfo = new PageInfo<>(bookList);
|
||||
pageInfo.setTotal(total.longValue());
|
||||
pageInfo.setPageNum(page);
|
||||
pageInfo.setPageSize(pageSize);
|
||||
return pageInfo;
|
||||
}
|
||||
throw new BusinessException(ResponseStatus.ES_SEARCH_FAIL);
|
||||
}
|
||||
}
|
@ -1,16 +0,0 @@
|
||||
#FastDfs的配置 ====================================
|
||||
#读取inputsream阻塞时间
|
||||
fdfs:
|
||||
connect-timeout: 600
|
||||
so-timeout: 1500
|
||||
#tracker地址
|
||||
trackerList: 127.0.0.1:22122
|
||||
#缩略图配置
|
||||
thumbImage:
|
||||
height: 150
|
||||
width: 150
|
||||
#通过nginx 访问地址
|
||||
webUrl: http://127.0.0.1/
|
||||
#获取连接池最大数量
|
||||
pool.max-total: 200
|
||||
|
@ -1,9 +0,0 @@
|
||||
#OSS
|
||||
novel:
|
||||
file:
|
||||
endpoint: oss-cn-shenzhen.aliyuncs.com #不同的服务器,地址不同
|
||||
key-id: dhjjrue6767778878 #去OSS控制台获取
|
||||
key-secret: dssdkkrkelrkellk44554 #去OSS控制台获取
|
||||
bucket-name: novel #这个自己创建bucket时的命名,控制台创建也行,代码创建也行
|
||||
file-host: pic #文件路径
|
||||
web-url: https://xxyimg.oss-cn-hangzhou.aliyuncs.com #OSS文件的web访问地址
|
@ -4,19 +4,7 @@ server:
|
||||
spring:
|
||||
profiles:
|
||||
active: dev
|
||||
include: alipay,oss,fastdfs
|
||||
|
||||
|
||||
rabbitmq:
|
||||
enable: 0
|
||||
host: 127.0.0.1
|
||||
username: guest
|
||||
password: guest
|
||||
virtual-host: /novel-plus
|
||||
template:
|
||||
# 缺省的交换机名称,此处配置后,发送消息如果不指定交换机就会使用这个
|
||||
exchange: novel.exchange
|
||||
publisher-confirms: false
|
||||
include: alipay
|
||||
|
||||
|
||||
elasticsearch:
|
||||
@ -25,9 +13,7 @@ spring:
|
||||
jest:
|
||||
uris: http://127.0.0.1:9200
|
||||
|
||||
redisson:
|
||||
singleServerConfig:
|
||||
address: 127.0.0.1:6379
|
||||
|
||||
|
||||
jwt:
|
||||
secret: novel!#20191230
|
||||
|
@ -35,7 +35,7 @@
|
||||
</select>
|
||||
|
||||
<update id="addVisitCount" >
|
||||
update book set visit_count = visit_count + ${visitCount}
|
||||
update book set visit_count = visit_count + 1 , update_time = #{date}
|
||||
where id = #{bookId}
|
||||
</update>
|
||||
|
||||
@ -56,7 +56,6 @@
|
||||
select
|
||||
id,pic_url from book
|
||||
where pic_url like 'http%'
|
||||
and pic_url not like concat('%',#{localPicPrefix},'%')
|
||||
limit #{limit}
|
||||
</select>
|
||||
|
||||
|
8
pom.xml
8
pom.xml
@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.java2nb</groupId>
|
||||
<artifactId>novel</artifactId>
|
||||
<version>2.5.1</version>
|
||||
<version>2.2.0</version>
|
||||
<modules>
|
||||
<module>novel-common</module>
|
||||
<module>novel-front</module>
|
||||
@ -39,12 +39,6 @@
|
||||
<elasticsearch.version>6.2.2</elasticsearch.version>
|
||||
<jest.version>6.3.1</jest.version>
|
||||
<redis.version>1.4.1.RELEASE</redis.version>
|
||||
<redisson.version>3.12.5</redisson.version>
|
||||
<sharding.jdbc.version>3.0.0</sharding.jdbc.version>
|
||||
<aliyun-sdk-oss.version>2.4.0</aliyun-sdk-oss.version>
|
||||
<commons-fileupload.version>1.3.1</commons-fileupload.version>
|
||||
<fastdfs-client.version>1.26.1-RELEASE</fastdfs-client.version>
|
||||
<alipay-sdk-java.version>4.9.153.ALL</alipay-sdk-java.version>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
|
@ -1,2 +0,0 @@
|
||||
INSERT INTO `crawl_source` (`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES
|
||||
(6, '新笔趣阁', '{\n \"bookListUrl\": \"http://www.xbiquge.la/fenlei/{catId}_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"6\",\n \"catId6\": \"5\"\n },\n \"bookIdPatten\": \"<a\\\\s+href=\\\"http://www.xbiquge.la/(\\\\d+/\\\\d+)/\\\"\\\\s+target=\\\"_blank\\\">\",\n \"pagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">(\\\\d+)/\\\\d+</em>\",\n \"totalPagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">\\\\d+/(\\\\d+)</em>\",\n \"bookDetailUrl\": \"http://www.xbiquge.la/{bookId}/\",\n \"bookNamePatten\": \"<h1>([^/]+)</h1>\",\n \"authorNamePatten\": \"者:([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.xbiquge.la/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"bookStatusRule\": {},\n \"descStart\": \"<div id=\\\"intro\\\">\",\n \"descEnd\": \"</div>\",\n \"upadateTimePatten\": \"<p>最后更新:(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)</p>\",\n \"upadateTimeFormatPatten\": \"yyyy-MM-dd HH:mm:ss\",\n \"bookIndexUrl\": \"http://www.xbiquge.la/{bookId}/\",\n \"indexIdPatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/(\\\\d+)\\\\.html\'\\\\s+>[^/]+</a>\",\n \"indexNamePatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/\\\\d+\\\\.html\'\\\\s+>([^/]+)</a>\",\n \"bookContentUrl\": \"http://www.xbiquge.la/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\">\",\n \"contentEnd\": \"<p>\"\n}', 0, '2020-05-23 22:46:58', '2020-05-23 22:46:58');
|
@ -1691,122 +1691,4 @@ INSERT INTO `user_read_history` VALUES ('113', '1255060328322027520', '125467697
|
||||
INSERT INTO `user_read_history` VALUES ('117', '1255060328322027520', '1254946661743603712', '1254946914001629184', '2020-04-30 19:37:09', '2020-04-30 19:37:09');
|
||||
INSERT INTO `user_read_history` VALUES ('118', '1255060328322027520', '1254957312633352192', '3335449', '2020-04-30 19:37:36', '2020-04-30 19:37:36');
|
||||
|
||||
|
||||
|
||||
DROP TABLE IF EXISTS `book_content0`;
|
||||
CREATE TABLE `book_content0` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1155 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content1
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content1`;
|
||||
CREATE TABLE `book_content1` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=406 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content2
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content2`;
|
||||
CREATE TABLE `book_content2` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1222 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content3
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content3`;
|
||||
CREATE TABLE `book_content3` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=410 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content4
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content4`;
|
||||
CREATE TABLE `book_content4` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1188 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content5
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content5`;
|
||||
CREATE TABLE `book_content5` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=416 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content6
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content6`;
|
||||
CREATE TABLE `book_content6` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1180 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content7
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content7`;
|
||||
CREATE TABLE `book_content7` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=404 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content8
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content8`;
|
||||
CREATE TABLE `book_content8` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1134 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for book_content9
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `book_content9`;
|
||||
CREATE TABLE `book_content9` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
|
||||
`content` mediumtext COMMENT '小说章节内容',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n \"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"7\",\n \"catId6\": \"6\",\n \"catId7\": \"8\"\n },\n \"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n \"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n \"totalPagePatten\": \"第\\\\d+/(\\\\d+)页\",\n \"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n \"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n \"authorNamePatten\": \"<p>作者:([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"statusPatten\": \"<p>状态:([^/]+)</p>\",\n \"bookStatusRule\": {\n \"连载中\": 0,\n \"完本\": 1\n },\n \"descStart\": \"<div class=\\\"intro_info\\\">\",\n \"descEnd\": \"最新章节推荐地址\",\n \"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n \"bookIndexStart\": \"<dt>《\",\n \"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n \"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n \"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n \"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;
|
Reference in New Issue
Block a user