201206030/novel
1
0
Fork 0
mirror of https://github.com/201206030/novel.git synced 2025-04-27 07:30:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: 集成 Elasticsearch 8.2.0

Browse Source
This commit is contained in:
xiongxiaoyang 2022-05-23 21:57:07 +08:00
parent 61d261d277
commit d45dc3d015
10 changed files with 532 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -175,7 +175,7 @@ git clone https://gitee.com/novel_dev_team/novel.git
1. 新建数据库（建议 novel）
2. 解压后端源码`sql/novel.sql.zip`压缩包，得到数据库结构文件`novel_struc.sql`和数据库小说数据文件`novel_data.sql`
2. 解压后端源码`doc/sql/novel.sql.zip`压缩包，得到数据库结构文件`novel_struc.sql`和数据库小说数据文件`novel_data.sql`
3. 导入`novel_struct.sql`数据库结构文件

62
doc/es/book.http Normal file
View File

@ -0,0 +1,62 @@
PUT /book
{
"mappings" : {
"properties" : {
"id" : {
"type" : "long"
},
"authorId" : {
"type" : "long"
},
"authorName" : {
"type" : "text",
"analyzer": "ik_smart"
},
"bookName" : {
"type" : "text",
"analyzer": "ik_smart"
},
"bookDesc" : {
"type" : "text",
"analyzer": "ik_smart"
},
"bookStatus" : {
"type" : "short"
},
"categoryId" : {
"type" : "integer"
},
"categoryName" : {
"type" : "text",
"analyzer": "ik_smart"
},
"lastChapterId" : {
"type" : "long"
},
"lastChapterName" : {
"type" : "text",
"analyzer": "ik_smart"
},
"lastChapterUpdateTime" : {
"type": "keyword"
},
"picUrl" : {
"type" : "keyword",
"index" : false,
"doc_values" : false
},
"score" : {
"type" : "integer"
},
"wordCount" : {
"type" : "integer"
},
"workDirection" : {
"type" : "short"
},
"visitCount" : {
"type": "long"
}
}
}
}

0
sql/novel.sql.zip → doc/sql/novel.sql.zip
View File

344
pom.xml
View File

@ -1,176 +1,192 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>io.github.xxyopen</groupId>
<artifactId>novel</artifactId>
<version>3.0.0</version>
<name>novel</name>
<description>Spring Boot 3 + Vue 3 构建的前后端分离小说系统</description>
<properties>
<java.version>17</java.version>
<mybatis-plus.version>3.5.1</mybatis-plus.version>
<spring.version>6.0.0-SNAPSHOT</spring.version>
<jjwt.version>0.11.5</jjwt.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<!-- Exclude the Tomcat dependency -->
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>io.github.xxyopen</groupId>
<artifactId>novel</artifactId>
<version>3.0.0</version>
<name>novel</name>
<description>Spring Boot 3 + Vue 3 构建的前后端分离小说系统</description>
<properties>
<java.version>17</java.version>
<mybatis-plus.version>3.5.1</mybatis-plus.version>
<spring.version>6.0.0-SNAPSHOT</spring.version>
<jjwt.version>0.11.5</jjwt.version>
<elasticsearch.version>8.2.0</elasticsearch.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<!-- Exclude the Tomcat dependency -->
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Use Undertow instead -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-undertow</artifactId>
</dependency>
<!-- Use Undertow instead -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-undertow</artifactId>
</dependency>
<!-- mybatis-plus -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>${mybatis-plus.version}</version>
</dependency>
<!-- mybatis-plus -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>${mybatis-plus.version}</version>
</dependency>
<!-- mybatis-plus 代码生成器 -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-generator</artifactId>
<version>${mybatis-plus.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-engine-core</artifactId>
<version>2.3</version>
<scope>test</scope>
</dependency>
<!-- mybatis-plus 代码生成器 -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-generator</artifactId>
<version>${mybatis-plus.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-engine-core</artifactId>
<version>2.3</version>
<scope>test</scope>
</dependency>
<!-- 缓存相关 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<!-- 缓存相关 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<!-- JWT 相关 -->
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-api</artifactId>
<version>${jjwt.version}</version>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-impl</artifactId>
<version>${jjwt.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-jackson</artifactId> <!-- or jjwt-gson if Gson is preferred -->
<version>${jjwt.version}</version>
<scope>runtime</scope>
</dependency>
<!-- JWT 相关 -->
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-api</artifactId>
<version>${jjwt.version}</version>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-impl</artifactId>
<version>${jjwt.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-jackson</artifactId>
<version>${jjwt.version}</version>
<scope>runtime</scope>
</dependency>
<!-- 请求参数校验相关 -->
<dependency>
<groupId>org.hibernate.validator</groupId>
<artifactId>hibernate-validator</artifactId>
</dependency>
<!-- 请求参数校验相关 -->
<dependency>
<groupId>org.hibernate.validator</groupId>
<artifactId>hibernate-validator</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<!-- elasticsearch 相关 -->
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<excludes>
<exclude>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<releases>
<enabled>false</enabled>
</releases>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<releases>
<enabled>false</enabled>
</releases>
</pluginRepository>
</pluginRepositories>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<excludes>
<exclude>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<releases>
<enabled>false</enabled>
</releases>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<releases>
<enabled>false</enabled>
</releases>
</pluginRepository>
</pluginRepositories>
</project>

2
src/main/java/io/github/xxyopen/novel/NovelApplication.java
View File

@ -9,12 +9,14 @@ import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
import java.util.Map;
@SpringBootApplication
@MapperScan("io.github.xxyopen.novel.dao.mapper")
@EnableCaching
@EnableScheduling
@Slf4j
public class NovelApplication {

35
src/main/java/io/github/xxyopen/novel/core/config/EsConfig.java Normal file
View File

@ -0,0 +1,35 @@
package io.github.xxyopen.novel.core.config;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import lombok.RequiredArgsConstructor;
import org.elasticsearch.client.RestClient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* elasticsearch 相关配置
*
* @author xiongxiaoyang
* @date 2022/5/23
*/
@Configuration
@ConditionalOnProperty(prefix = "spring.elasticsearch", name = "enable", havingValue = "true")
@RequiredArgsConstructor
public class EsConfig {
@Bean
public ElasticsearchClient elasticsearchClient(RestClient restClient) {
// Create the transport with a Jackson mapper
ElasticsearchTransport transport = new RestClientTransport(
restClient, new JacksonJsonpMapper());
// And create the API client
return new ElasticsearchClient(transport);
}
}

28
src/main/java/io/github/xxyopen/novel/core/constant/EsConsts.java Normal file
View File

@ -0,0 +1,28 @@
package io.github.xxyopen.novel.core.constant;
import lombok.Getter;
/**
* elasticsearch 相关常量
*
* @author xiongxiaoyang
* @date 2022/5/23
*/
public class EsConsts {
/**
* ES 索引枚举类
*/
@Getter
public enum IndexEnum {
BOOK("book");
IndexEnum(String name) {
this.name = name;
}
private String name;
}
}

109
src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java Normal file
View File

@ -0,0 +1,109 @@
package io.github.xxyopen.novel.core.task;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch.core.BulkRequest;
import co.elastic.clients.elasticsearch.core.BulkResponse;
import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import io.github.xxyopen.novel.core.constant.DatabaseConsts;
import io.github.xxyopen.novel.core.constant.EsConsts;
import io.github.xxyopen.novel.dao.entity.BookInfo;
import io.github.xxyopen.novel.dao.mapper.BookInfoMapper;
import io.github.xxyopen.novel.dto.es.EsBookDto;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 小说数据同步到 elasticsearch 任务
*
* @author xiongxiaoyang
* @date 2022/5/23
*/
@ConditionalOnProperty(prefix = "spring.elasticsearch", name = "enable", havingValue = "true")
@Component
@RequiredArgsConstructor
@Slf4j
public class BookToEsTask {
private final BookInfoMapper bookInfoMapper;
private final ElasticsearchClient elasticsearchClient;
/**
* 每月凌晨做一次全量数据同步
*/
@SneakyThrows
@Scheduled(cron = "0 0 0 1 * ?")
public void saveToEs() {
QueryWrapper<BookInfo> queryWrapper = new QueryWrapper<>();
List<BookInfo> bookInfos;
long maxId = 0;
for(;;) {
queryWrapper.clear();
queryWrapper
.orderByAsc(DatabaseConsts.CommonColumnEnum.ID.getName())
.gt(DatabaseConsts.CommonColumnEnum.ID.getName(), maxId)
.last(DatabaseConsts.SqlEnum.LIMIT_30.getSql());
bookInfos = bookInfoMapper.selectList(queryWrapper);
if (bookInfos.isEmpty()) {
break;
}
BulkRequest.Builder br = new BulkRequest.Builder();
for (BookInfo book : bookInfos) {
EsBookDto esBook = buildEsBook(book);
br.operations(op -> op
.index(idx -> idx
.index(EsConsts.IndexEnum.BOOK.getName())
.id(book.getId().toString())
.document(esBook)
)
);
maxId = book.getId();
}
BulkResponse result = elasticsearchClient.bulk(br.build());
// Log errors, if any
if (result.errors()) {
log.error("Bulk had errors");
for (BulkResponseItem item : result.items()) {
if (item.error() != null) {
log.error(item.error().reason());
}
}
}
}
}
private EsBookDto buildEsBook(BookInfo book) {
return EsBookDto.builder()
.id(book.getId())
.categoryId(book.getCategoryId())
.categoryName(book.getCategoryName())
.bookDesc(book.getBookDesc())
.bookName(book.getBookName())
.authorId(book.getAuthorId())
.authorName(book.getAuthorName())
.bookStatus(book.getBookStatus())
.commentCount(book.getCommentCount())
.isVip(book.getIsVip())
.score(book.getScore())
.visitCount(book.getVisitCount())
.wordCount(book.getWordCount())
.workDirection(book.getWorkDirection())
.lastChapterId(book.getLastChapterId())
.lastChapterName(book.getLastChapterName())
.lastChapterUpdateTime(book.getLastChapterUpdateTime())
.build();
}
}

108
src/main/java/io/github/xxyopen/novel/dto/es/EsBookDto.java Normal file
View File

@ -0,0 +1,108 @@
package io.github.xxyopen.novel.dto.es;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
import lombok.Builder;
import lombok.Data;
import java.time.LocalDateTime;
/**
* Elasticsearch 存储小说 DTO
* @author xiongxiaoyang
* @date 2022/5/23
*/
@Data
@Builder
public class EsBookDto {
/**
* id
*/
private Long id;
/**
* 作品方向;0-男频 1-女频
*/
private Integer workDirection;
/**
* 类别ID
*/
private Long categoryId;
/**
* 类别名
*/
private String categoryName;
/**
* 小说名
*/
private String bookName;
/**
* 作家id
*/
private Long authorId;
/**
* 作家名
*/
private String authorName;
/**
* 书籍描述
*/
private String bookDesc;
/**
* 评分;总分:10 ，真实评分 = score/10
*/
private Integer score;
/**
* 书籍状态;0-连载中 1-已完结
*/
private Integer bookStatus;
/**
* 点击量
*/
private Long visitCount;
/**
* 总字数
*/
private Integer wordCount;
/**
* 评论数
*/
private Integer commentCount;
/**
* 最新章节ID
*/
private Long lastChapterId;
/**
* 最新章节名
*/
private String lastChapterName;
/**
* 最新章节更新时间
*/
@JsonDeserialize(using = LocalDateTimeDeserializer.class)
@JsonSerialize(using = LocalDateTimeSerializer.class)
private LocalDateTime lastChapterUpdateTime;
/**
* 是否收费;1-收费 0-免费
*/
private Integer isVip;
}

8
src/main/resources/application.yml
View File

@ -41,7 +41,13 @@ spring:
config:
activate:
on-profile: dev
elasticsearch:
# 是否开启 elasticsearch 搜索引擎功能：true-开启 false-不开启
enable: false
uris:
- https://my-deployment-ce7ca3.es.us-central1.gcp.cloud.es.io:9243
username: elastic
password: qTjgYVKSuExX6tWAsDuvuvwl
---
spring: