From 220068cd3ae993dabaf15f00bcefbc53d5ca2ba6 Mon Sep 17 00:00:00 2001 From: xiongxiaoyang <773861846@qq.com> Date: Tue, 31 May 2022 14:03:43 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=9B=86=E6=88=90=E5=88=86=E5=B8=83?= =?UTF-8?q?=E5=BC=8F=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=20XXL-JOB?= =?UTF-8?q?=EF=BC=8C=20=E4=BC=98=E5=8C=96=20Elasticsearch=20=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=90=8C=E6=AD=A5=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 7 ++ .../novel/core/config/XxlJobConfig.java | 44 +++++++++++ .../core/listener/RabbitQueueListener.java | 2 +- .../xxyopen/novel/core/task/BookToEsTask.java | 78 ++++++++++--------- .../service/impl/EsSearchServiceImpl.java | 6 ++ src/main/resources/application.yml | 15 ++++ 6 files changed, 115 insertions(+), 37 deletions(-) create mode 100644 src/main/java/io/github/xxyopen/novel/core/config/XxlJobConfig.java diff --git a/pom.xml b/pom.xml index 47b6466..675e55a 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,7 @@ 6.0.0-SNAPSHOT 0.11.5 8.2.0 + 2.3.1 @@ -116,6 +117,12 @@ spring-boot-starter-amqp + + com.xuxueli + xxl-job-core + ${xxl-job.version} + + mysql mysql-connector-java diff --git a/src/main/java/io/github/xxyopen/novel/core/config/XxlJobConfig.java b/src/main/java/io/github/xxyopen/novel/core/config/XxlJobConfig.java new file mode 100644 index 0000000..4084cf2 --- /dev/null +++ b/src/main/java/io/github/xxyopen/novel/core/config/XxlJobConfig.java @@ -0,0 +1,44 @@ +package io.github.xxyopen.novel.core.config; + +import com.xxl.job.core.executor.impl.XxlJobSpringExecutor; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * XXL-JOB 配置类 + * + * @author xiongxiaoyang + * @date 2022/5/31 + */ +@Configuration +@Slf4j +public class XxlJobConfig { + + @Value("${xxl.job.admin.addresses}") + private String adminAddresses; + + @Value("${xxl.job.accessToken}") + private String accessToken; + + @Value("${xxl.job.executor.appname}") + private String appname; + + @Value("${xxl.job.executor.logpath}") + private String logPath; + + @Bean + public XxlJobSpringExecutor xxlJobExecutor() { + log.info(">>>>>>>>>>> xxl-job config init."); + XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor(); + xxlJobSpringExecutor.setAdminAddresses(adminAddresses); + xxlJobSpringExecutor.setAccessToken(accessToken); + xxlJobSpringExecutor.setAppname(appname); + xxlJobSpringExecutor.setLogPath(logPath); + return xxlJobSpringExecutor; + } + +} \ No newline at end of file diff --git a/src/main/java/io/github/xxyopen/novel/core/listener/RabbitQueueListener.java b/src/main/java/io/github/xxyopen/novel/core/listener/RabbitQueueListener.java index 55d82f2..1d0971f 100644 --- a/src/main/java/io/github/xxyopen/novel/core/listener/RabbitQueueListener.java +++ b/src/main/java/io/github/xxyopen/novel/core/listener/RabbitQueueListener.java @@ -21,7 +21,7 @@ import org.springframework.stereotype.Component; * @date 2022/5/25 */ @Component -@ConditionalOnProperty(prefix = "spring.elasticsearch", name = "enable", havingValue = "true") +@ConditionalOnProperty(prefix = "spring", name = {"elasticsearch.enable","amqp.enable"}, havingValue = "true") @RequiredArgsConstructor @Slf4j public class RabbitQueueListener { diff --git a/src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java b/src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java index a456e38..69704c5 100644 --- a/src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java +++ b/src/main/java/io/github/xxyopen/novel/core/task/BookToEsTask.java @@ -6,6 +6,8 @@ import co.elastic.clients.elasticsearch.core.BulkRequest; import co.elastic.clients.elasticsearch.core.BulkResponse; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.handler.annotation.XxlJob; import io.github.xxyopen.novel.core.constant.DatabaseConsts; import io.github.xxyopen.novel.core.constant.EsConsts; import io.github.xxyopen.novel.dao.entity.BookInfo; @@ -15,7 +17,6 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import java.util.List; @@ -40,49 +41,54 @@ public class BookToEsTask { * 每月凌晨做一次全量数据同步 */ @SneakyThrows - @Scheduled(cron = "0 0 0 1 * ?") - public void saveToEs() { + @XxlJob("saveToEsJobHandler") + public ReturnT saveToEs() { - QueryWrapper queryWrapper = new QueryWrapper<>(); - List bookInfos; - long maxId = 0; - for(;;) { - queryWrapper.clear(); - queryWrapper - .orderByAsc(DatabaseConsts.CommonColumnEnum.ID.getName()) - .gt(DatabaseConsts.CommonColumnEnum.ID.getName(), maxId) - .last(DatabaseConsts.SqlEnum.LIMIT_30.getSql()); - bookInfos = bookInfoMapper.selectList(queryWrapper); - if (bookInfos.isEmpty()) { - break; - } - BulkRequest.Builder br = new BulkRequest.Builder(); + try { + QueryWrapper queryWrapper = new QueryWrapper<>(); + List bookInfos; + long maxId = 0; + for (; ; ) { + queryWrapper.clear(); + queryWrapper + .orderByAsc(DatabaseConsts.CommonColumnEnum.ID.getName()) + .gt(DatabaseConsts.CommonColumnEnum.ID.getName(), maxId) + .gt(DatabaseConsts.BookTable.COLUMN_WORD_COUNT, 0) + .last(DatabaseConsts.SqlEnum.LIMIT_30.getSql()); + bookInfos = bookInfoMapper.selectList(queryWrapper); + if (bookInfos.isEmpty()) { + break; + } + BulkRequest.Builder br = new BulkRequest.Builder(); - for (BookInfo book : bookInfos) { - br.operations(op -> op - .index(idx -> idx - .index(EsConsts.BookIndex.INDEX_NAME) - .id(book.getId().toString()) - .document(EsBookDto.build(book)) - ) - ).timeout(Time.of(t -> t.time("10s"))); - maxId = book.getId(); - } + for (BookInfo book : bookInfos) { + br.operations(op -> op + .index(idx -> idx + .index(EsConsts.BookIndex.INDEX_NAME) + .id(book.getId().toString()) + .document(EsBookDto.build(book)) + ) + ).timeout(Time.of(t -> t.time("10s"))); + maxId = book.getId(); + } - BulkResponse result = elasticsearchClient.bulk(br.build()); + BulkResponse result = elasticsearchClient.bulk(br.build()); - // Log errors, if any - if (result.errors()) { - log.error("Bulk had errors"); - for (BulkResponseItem item : result.items()) { - if (item.error() != null) { - log.error(item.error().reason()); + // Log errors, if any + if (result.errors()) { + log.error("Bulk had errors"); + for (BulkResponseItem item : result.items()) { + if (item.error() != null) { + log.error(item.error().reason()); + } } } } - + return ReturnT.SUCCESS; + } catch (Exception e) { + log.error(e.getMessage(), e); + return ReturnT.FAIL; } - } } diff --git a/src/main/java/io/github/xxyopen/novel/service/impl/EsSearchServiceImpl.java b/src/main/java/io/github/xxyopen/novel/service/impl/EsSearchServiceImpl.java index 705f467..d977947 100644 --- a/src/main/java/io/github/xxyopen/novel/service/impl/EsSearchServiceImpl.java +++ b/src/main/java/io/github/xxyopen/novel/service/impl/EsSearchServiceImpl.java @@ -109,6 +109,12 @@ public class EsSearchServiceImpl implements SearchService { BoolQuery boolQuery = BoolQuery.of(b -> { + // 只查有字数的小说 + b.must(RangeQuery.of(m -> m + .field(EsConsts.BookIndex.FIELD_WORD_COUNT) + .gt(JsonData.of(0)) + )._toQuery()); + if (!StringUtils.isBlank(condition.getKeyword())) { // 关键词匹配 b.must((q -> q.multiMatch(t -> t diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 41e220b..2492763 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -58,6 +58,21 @@ spring: # 第一次和第二次重试之间的持续时间 initial-interval: "3s" +# XXL-JOB 配置 +xxl: + job: + admin: + ### 调度中心部署根地址 [选填]：如调度中心集群部署存在多个地址则用逗号分隔。执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调"；为空则关闭自动注册； + addresses: http://127.0.0.1:8080/xxl-job-admin + executor: + ### 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册 + appname: xxl-job-executor-novel + ### 执行器运行日志文件存储磁盘路径 [选填] ：需要对该路径拥有读写权限；为空则使用默认路径； + logpath: logs/xxl-job/jobhandler + ### xxl-job, access token + accessToken: 123 + + --- spring: config: