From d45dc3d015b4e409eca23037cfbcf8b4ba1a65ca Mon Sep 17 00:00:00 2001 From: xiongxiaoyang <773861846@qq.com> Date: Mon, 23 May 2022 21:57:07 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=9B=86=E6=88=90=20Elasticsearch=208.?= =?UTF-8?q?2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- doc/es/book.http | 62 ++++ {sql => doc/sql}/novel.sql.zip | Bin pom.xml | 344 +++++++++--------- .../xxyopen/novel/NovelApplication.java | 2 + .../xxyopen/novel/core/config/EsConfig.java | 35 ++ .../xxyopen/novel/core/constant/EsConsts.java | 28 ++ .../xxyopen/novel/core/task/BookToEsTask.java | 109 ++++++ .../xxyopen/novel/dto/es/EsBookDto.java | 108 ++++++ src/main/resources/application.yml | 8 +- 10 files changed, 532 insertions(+), 166 deletions(-) create mode 100644 doc/es/book.http rename {sql => doc/sql}/novel.sql.zip (100%) create mode 100644 src/main/java/io/github/xxyopen/novel/core/config/EsConfig.java create mode 100644 src/main/java/io/github/xxyopen/novel/core/constant/EsConsts.java create mode 100644 src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java create mode 100644 src/main/java/io/github/xxyopen/novel/dto/es/EsBookDto.java diff --git a/README.md b/README.md index 3f59a6b..4617b6c 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ git clone https://gitee.com/novel_dev_team/novel.git 1. 新建数据库（建议 novel） - 2. 解压后端源码`sql/novel.sql.zip`压缩包，得到数据库结构文件`novel_struc.sql`和数据库小说数据文件`novel_data.sql` + 2. 解压后端源码`doc/sql/novel.sql.zip`压缩包，得到数据库结构文件`novel_struc.sql`和数据库小说数据文件`novel_data.sql` 3. 导入`novel_struct.sql`数据库结构文件 diff --git a/doc/es/book.http b/doc/es/book.http new file mode 100644 index 0000000..e80fd3d --- /dev/null +++ b/doc/es/book.http @@ -0,0 +1,62 @@ +PUT /book +{ + "mappings" : { + "properties" : { + "id" : { + "type" : "long" + }, + "authorId" : { + "type" : "long" + }, + "authorName" : { + "type" : "text", + "analyzer": "ik_smart" + }, + "bookName" : { + "type" : "text", + "analyzer": "ik_smart" + }, + "bookDesc" : { + "type" : "text", + "analyzer": "ik_smart" + }, + "bookStatus" : { + "type" : "short" + }, + "categoryId" : { + "type" : "integer" + }, + "categoryName" : { + "type" : "text", + "analyzer": "ik_smart" + }, + "lastChapterId" : { + "type" : "long" + }, + "lastChapterName" : { + "type" : "text", + "analyzer": "ik_smart" + }, + "lastChapterUpdateTime" : { + "type": "keyword" + }, + "picUrl" : { + "type" : "keyword", + "index" : false, + "doc_values" : false + }, + "score" : { + "type" : "integer" + }, + "wordCount" : { + "type" : "integer" + }, + "workDirection" : { + "type" : "short" + }, + "visitCount" : { + "type": "long" + } + } + } +} \ No newline at end of file diff --git a/sql/novel.sql.zip b/doc/sql/novel.sql.zip similarity index 100% rename from sql/novel.sql.zip rename to doc/sql/novel.sql.zip diff --git a/pom.xml b/pom.xml index 608939e..347f5e1 100644 --- a/pom.xml +++ b/pom.xml @@ -1,176 +1,192 @@ - 4.0.0 - - org.springframework.boot - spring-boot-starter-parent - 3.0.0-SNAPSHOT - - - io.github.xxyopen - novel - 3.0.0 - novel - Spring Boot 3 + Vue 3 构建的前后端分离小说系统 - - 17 - 3.5.1 - 6.0.0-SNAPSHOT - 0.11.5 - - - - org.springframework.boot - spring-boot-starter-web - - - - org.springframework.boot - spring-boot-starter-tomcat - - - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.0.0-SNAPSHOT + + + io.github.xxyopen + novel + 3.0.0 + novel + Spring Boot 3 + Vue 3 构建的前后端分离小说系统 + + 17 + 3.5.1 + 6.0.0-SNAPSHOT + 0.11.5 + 8.2.0 + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-tomcat + + + - - - org.springframework.boot - spring-boot-starter-undertow - + + + org.springframework.boot + spring-boot-starter-undertow + - - - com.baomidou - mybatis-plus-boot-starter - ${mybatis-plus.version} - + + + com.baomidou + mybatis-plus-boot-starter + ${mybatis-plus.version} + - - - com.baomidou - mybatis-plus-generator - ${mybatis-plus.version} - test - - - org.apache.velocity - velocity-engine-core - 2.3 - test - + + + com.baomidou + mybatis-plus-generator + ${mybatis-plus.version} + test + + + org.apache.velocity + velocity-engine-core + 2.3 + test + - - - org.springframework.boot - spring-boot-starter-cache - - - org.springframework.boot - spring-boot-starter-data-redis - - - com.github.ben-manes.caffeine - caffeine - + + + org.springframework.boot + spring-boot-starter-cache + + + org.springframework.boot + spring-boot-starter-data-redis + + + com.github.ben-manes.caffeine + caffeine + - - - io.jsonwebtoken - jjwt-api - ${jjwt.version} - - - io.jsonwebtoken - jjwt-impl - ${jjwt.version} - runtime - - - io.jsonwebtoken - jjwt-jackson - ${jjwt.version} - runtime - + + + io.jsonwebtoken + jjwt-api + ${jjwt.version} + + + io.jsonwebtoken + jjwt-impl + ${jjwt.version} + runtime + + + io.jsonwebtoken + jjwt-jackson + ${jjwt.version} + runtime + - - - org.hibernate.validator - hibernate-validator - + + + org.hibernate.validator + hibernate-validator + - - mysql - mysql-connector-java - runtime - - - org.springframework.boot - spring-boot-configuration-processor - true - - - org.projectlombok - lombok - true - - - org.springframework.boot - spring-boot-starter-test - test - - + + + co.elastic.clients + elasticsearch-java + ${elasticsearch.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + - - - - org.springframework.boot - spring-boot-maven-plugin - - - - org.projectlombok - lombok - - - - - - - - - spring-milestones - Spring Milestones - https://repo.spring.io/milestone - - false - - - - spring-snapshots - Spring Snapshots - https://repo.spring.io/snapshot - - false - - - - - - spring-milestones - Spring Milestones - https://repo.spring.io/milestone - - false - - - - spring-snapshots - Spring Snapshots - https://repo.spring.io/snapshot - - false - - - + + mysql + mysql-connector-java + runtime + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.projectlombok + lombok + true + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + false + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + false + + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + false + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + false + + + diff --git a/src/main/java/io/github/xxyopen/novel/NovelApplication.java b/src/main/java/io/github/xxyopen/novel/NovelApplication.java index 0de7b9c..5f5d082 100644 --- a/src/main/java/io/github/xxyopen/novel/NovelApplication.java +++ b/src/main/java/io/github/xxyopen/novel/NovelApplication.java @@ -9,12 +9,14 @@ import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; import java.util.Map; @SpringBootApplication @MapperScan("io.github.xxyopen.novel.dao.mapper") @EnableCaching +@EnableScheduling @Slf4j public class NovelApplication { diff --git a/src/main/java/io/github/xxyopen/novel/core/config/EsConfig.java b/src/main/java/io/github/xxyopen/novel/core/config/EsConfig.java new file mode 100644 index 0000000..0603fe4 --- /dev/null +++ b/src/main/java/io/github/xxyopen/novel/core/config/EsConfig.java @@ -0,0 +1,35 @@ +package io.github.xxyopen.novel.core.config; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import lombok.RequiredArgsConstructor; +import org.elasticsearch.client.RestClient; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * elasticsearch 相关配置 + * + * @author xiongxiaoyang + * @date 2022/5/23 + */ +@Configuration +@ConditionalOnProperty(prefix = "spring.elasticsearch", name = "enable", havingValue = "true") +@RequiredArgsConstructor +public class EsConfig { + + @Bean + public ElasticsearchClient elasticsearchClient(RestClient restClient) { + + // Create the transport with a Jackson mapper + ElasticsearchTransport transport = new RestClientTransport( + restClient, new JacksonJsonpMapper()); + + // And create the API client + return new ElasticsearchClient(transport); + } + +} diff --git a/src/main/java/io/github/xxyopen/novel/core/constant/EsConsts.java b/src/main/java/io/github/xxyopen/novel/core/constant/EsConsts.java new file mode 100644 index 0000000..d4b9a26 --- /dev/null +++ b/src/main/java/io/github/xxyopen/novel/core/constant/EsConsts.java @@ -0,0 +1,28 @@ +package io.github.xxyopen.novel.core.constant; + +import lombok.Getter; + +/** + * elasticsearch 相关常量 + * + * @author xiongxiaoyang + * @date 2022/5/23 + */ +public class EsConsts { + + /** + * ES 索引枚举类 + */ + @Getter + public enum IndexEnum { + + BOOK("book"); + + IndexEnum(String name) { + this.name = name; + } + + private String name; + + } +} diff --git a/src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java b/src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java new file mode 100644 index 0000000..3c9be46 --- /dev/null +++ b/src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java @@ -0,0 +1,109 @@ +package io.github.xxyopen.novel.core.task; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import io.github.xxyopen.novel.core.constant.DatabaseConsts; +import io.github.xxyopen.novel.core.constant.EsConsts; +import io.github.xxyopen.novel.dao.entity.BookInfo; +import io.github.xxyopen.novel.dao.mapper.BookInfoMapper; +import io.github.xxyopen.novel.dto.es.EsBookDto; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * 小说数据同步到 elasticsearch 任务 + * + * @author xiongxiaoyang + * @date 2022/5/23 + */ +@ConditionalOnProperty(prefix = "spring.elasticsearch", name = "enable", havingValue = "true") +@Component +@RequiredArgsConstructor +@Slf4j +public class BookToEsTask { + + private final BookInfoMapper bookInfoMapper; + + private final ElasticsearchClient elasticsearchClient; + + /** + * 每月凌晨做一次全量数据同步 + */ + @SneakyThrows + @Scheduled(cron = "0 0 0 1 * ?") + public void saveToEs() { + + QueryWrapper queryWrapper = new QueryWrapper<>(); + List bookInfos; + long maxId = 0; + for(;;) { + queryWrapper.clear(); + queryWrapper + .orderByAsc(DatabaseConsts.CommonColumnEnum.ID.getName()) + .gt(DatabaseConsts.CommonColumnEnum.ID.getName(), maxId) + .last(DatabaseConsts.SqlEnum.LIMIT_30.getSql()); + bookInfos = bookInfoMapper.selectList(queryWrapper); + if (bookInfos.isEmpty()) { + break; + } + BulkRequest.Builder br = new BulkRequest.Builder(); + + for (BookInfo book : bookInfos) { + EsBookDto esBook = buildEsBook(book); + br.operations(op -> op + .index(idx -> idx + .index(EsConsts.IndexEnum.BOOK.getName()) + .id(book.getId().toString()) + .document(esBook) + ) + ); + maxId = book.getId(); + } + + BulkResponse result = elasticsearchClient.bulk(br.build()); + + // Log errors, if any + if (result.errors()) { + log.error("Bulk had errors"); + for (BulkResponseItem item : result.items()) { + if (item.error() != null) { + log.error(item.error().reason()); + } + } + } + + } + + } + + private EsBookDto buildEsBook(BookInfo book) { + return EsBookDto.builder() + .id(book.getId()) + .categoryId(book.getCategoryId()) + .categoryName(book.getCategoryName()) + .bookDesc(book.getBookDesc()) + .bookName(book.getBookName()) + .authorId(book.getAuthorId()) + .authorName(book.getAuthorName()) + .bookStatus(book.getBookStatus()) + .commentCount(book.getCommentCount()) + .isVip(book.getIsVip()) + .score(book.getScore()) + .visitCount(book.getVisitCount()) + .wordCount(book.getWordCount()) + .workDirection(book.getWorkDirection()) + .lastChapterId(book.getLastChapterId()) + .lastChapterName(book.getLastChapterName()) + .lastChapterUpdateTime(book.getLastChapterUpdateTime()) + .build(); + } +} diff --git a/src/main/java/io/github/xxyopen/novel/dto/es/EsBookDto.java b/src/main/java/io/github/xxyopen/novel/dto/es/EsBookDto.java new file mode 100644 index 0000000..7d77fea --- /dev/null +++ b/src/main/java/io/github/xxyopen/novel/dto/es/EsBookDto.java @@ -0,0 +1,108 @@ +package io.github.xxyopen.novel.dto.es; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer; +import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; +import lombok.Builder; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Elasticsearch 存储小说 DTO + * @author xiongxiaoyang + * @date 2022/5/23 + */ +@Data +@Builder +public class EsBookDto { + + /** + * id + */ + private Long id; + + /** + * 作品方向;0-男频 1-女频 + */ + private Integer workDirection; + + /** + * 类别ID + */ + private Long categoryId; + + /** + * 类别名 + */ + private String categoryName; + + /** + * 小说名 + */ + private String bookName; + + /** + * 作家id + */ + private Long authorId; + + /** + * 作家名 + */ + private String authorName; + + /** + * 书籍描述 + */ + private String bookDesc; + + /** + * 评分;总分:10 ，真实评分 = score/10 + */ + private Integer score; + + /** + * 书籍状态;0-连载中 1-已完结 + */ + private Integer bookStatus; + + /** + * 点击量 + */ + private Long visitCount; + + /** + * 总字数 + */ + private Integer wordCount; + + /** + * 评论数 + */ + private Integer commentCount; + + /** + * 最新章节ID + */ + private Long lastChapterId; + + /** + * 最新章节名 + */ + private String lastChapterName; + + /** + * 最新章节更新时间 + */ + @JsonDeserialize(using = LocalDateTimeDeserializer.class) + @JsonSerialize(using = LocalDateTimeSerializer.class) + private LocalDateTime lastChapterUpdateTime; + + /** + * 是否收费;1-收费 0-免费 + */ + private Integer isVip; + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index ed2103b..23df130 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -41,7 +41,13 @@ spring: config: activate: on-profile: dev - + elasticsearch: + # 是否开启 elasticsearch 搜索引擎功能：true-开启 false-不开启 + enable: false + uris: + - https://my-deployment-ce7ca3.es.us-central1.gcp.cloud.es.io:9243 + username: elastic + password: qTjgYVKSuExX6tWAsDuvuvwl --- spring: