From 1bda806862068e5d787a246e078eda14f2f2ce1f Mon Sep 17 00:00:00 2001 From: xiongxiaoyang Date: Tue, 2 Jun 2020 22:32:33 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BC=95=E5=85=A5rabbitmq/=E5=BA=94=E7=94=A8?= =?UTF-8?q?=E8=A7=A3=E5=81=B6/=E6=B5=81=E9=87=8F=E5=89=8A=E5=B3=B0?= =?UTF-8?q?=EF=BC=8C=E5=BC=95=E5=85=A5redisson=E5=AE=9E=E7=8E=B0=E5=88=86?= =?UTF-8?q?=E5=B8=83=E5=BC=8F=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...=> nacos_config_export_20200602214520.zip} | Bin 6303 -> 6345 bytes novel-book/book-service/pom.xml | 3 - .../novel/book/controller/BookController.java | 5 +- .../book/listener/BookVisitAddListener.java | 70 ++++++++++++++++++ .../src/main/resources/bootstrap.yml | 12 +++ novel-common/pom.xml | 11 +++ .../novel/common/config/RabbitConfig.java | 62 ++++++++++++++++ .../src/main/resources/application-common.yml | 6 ++ .../search/listener/BookVisitAddListener.java | 67 +++++++++++++++++ .../search/schedule/BookToEsSchedule.java | 55 +++++++------- novel-search/src/main/resources/bootstrap.yml | 4 + 11 files changed, 264 insertions(+), 31 deletions(-) rename doc/yml/{nacos_config_export_20200602203745.zip => nacos_config_export_20200602214520.zip} (65%) create mode 100644 novel-book/book-service/src/main/java/com/java2nb/novel/book/listener/BookVisitAddListener.java create mode 100644 novel-common/src/main/java/com/java2nb/novel/common/config/RabbitConfig.java create mode 100644 novel-search/src/main/java/com/java2nb/novel/search/listener/BookVisitAddListener.java diff --git a/doc/yml/nacos_config_export_20200602203745.zip b/doc/yml/nacos_config_export_20200602214520.zip similarity index 65% rename from doc/yml/nacos_config_export_20200602203745.zip rename to doc/yml/nacos_config_export_20200602214520.zip index bdf8b71e8f669bf7dea64126fe4be1e2a9eb62a1..036063cb611b9fec50d4a61e715ea87871713afd 100644 GIT binary patch delta 792 zcmbPlc+!w3z?+#xgn@&DgJIR$jXXL`Osm#zwqiQZ2%@=JE`sTMtW03>N9-3s{K@xN zB{rYo3UHqKi5o|*@aW8FEnq)!;E#-i zbcWoKmUL-xo7tB?l$x&TImX|HGln`uSLJqIz*hR)8s;SGm1~TCVtx? zuhIJX&0kl~Pxo(%jGi=e_N;iWFH<%$3TPA*7B&X^7S_pWOzLVpsjY5mv2}&c))SWY z`ntxlLZzLWnyxF49(|#Ciff7=Lx49s$Et3{q+mt{hN(;pKt3}kJbXa`H~F@(RDJ(J zufq-^uHWm9GGFi9CdU19wx^XZ(`3&%Gef4kY3<6`(j@I%py=qj;acyM376)dJac2} z{sa$^yL-#^-rudRJW}&2Y6J6u3Aw+OEH=%OJixtkLxO=x_3!qy`htXSZGXc$EYn0- zZ!no3i`k;%G)KNY=gWNG8nq;so_81Jc-3drOYm_%4%FtBabe-Svy<^n(>kxY>Q=L_ zr7pb0K6QH0yjRPghMNYjZ`E0L`*X;8@uIq`0+qF!ZGEpCU+Q)5$>W*3&c6;_l_p*^ z@$X#y`>(ZDUze`^{b%RS)#3W~aX*fH6eyj|m8L9&wSWY@@8{Gw1Hd8 z<9tx~v`HRXw(%DO1A{g&kOI6JnM4?1DR8o|s3$1d#6hT)5bBGlH%QP?3@q0p2Ik!q z^92cNh`WQRLUC6RwL{#Ech%ZM0q`i8%q?N701|0KvQr*tHv+s6o}4czI=NOthD~1- GBn$xFf>wN9-3s{K@xN zB{rYo3zi=eF^;f0o3*-u&`*oZo$?O{z~jXC5dp zSh7@frlzGu&BySR!5fz`qyp#3aob=b> z_VIR{X}O+hd0BOu7lMzmIV@mc2=Hd-xN}>0rxGIrgAWskj|eSLFihqVk*fDU=yljZ zWY2F=ejD>`TO&@~&$*NTkRFFu!}_+o+LqdzJMi~rv_lC0#kr2ev<)q;a8lly#|GOnCp`mN~5wEZ5#?**rw z(l`1o51qLBNnmR09Q9w7KSR%5iPro4@}1Jf*{yTpulMZPd3Wy7SC74pzW?+x#x~`u zRp4gtogV+sO<%prXZ6{z`+IBh-mm`px?kLGYpl!MT~3ov{WzLC{e+TSx#8;@jIz~d zo~!i;K2-Q24GO0DEkcrvz+kcj;s9?(CJ_c$vYKow>Iq5<9T4g=gpw2U1_@@1f#tSA zc+BFyAi*GUcMvsO+!aJU5_jX>w&YL%JR&BWO4!PSM3C*2XFvwqCr^|xVq+Bri2(qd CW - - - diff --git a/novel-book/book-service/src/main/java/com/java2nb/novel/book/controller/BookController.java b/novel-book/book-service/src/main/java/com/java2nb/novel/book/controller/BookController.java index f4d137b..b2bc38d 100644 --- a/novel-book/book-service/src/main/java/com/java2nb/novel/book/controller/BookController.java +++ b/novel-book/book-service/src/main/java/com/java2nb/novel/book/controller/BookController.java @@ -13,6 +13,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.web.bind.annotation.*; import java.util.HashMap; @@ -34,6 +35,8 @@ public class BookController { private final BookService bookService; + private final RabbitTemplate rabbitTemplate; + /** * 小说分类列表查询接口 */ @@ -67,7 +70,7 @@ public class BookController { @ApiOperation("点击量新增接口") @PostMapping("addVisitCount") public ResultBean addVisitCount(@ApiParam("小说ID") @RequestParam("bookId") Long bookId) { - bookService.addVisitCount(bookId, 1); + rabbitTemplate.convertAndSend("ADD-BOOK-VISIT-EXCHANGE", null, bookId); return ResultBean.ok(); } diff --git a/novel-book/book-service/src/main/java/com/java2nb/novel/book/listener/BookVisitAddListener.java b/novel-book/book-service/src/main/java/com/java2nb/novel/book/listener/BookVisitAddListener.java new file mode 100644 index 0000000..7a0fb8a --- /dev/null +++ b/novel-book/book-service/src/main/java/com/java2nb/novel/book/listener/BookVisitAddListener.java @@ -0,0 +1,70 @@ +package com.java2nb.novel.book.listener; +import com.java2nb.novel.book.service.BookService; +import com.java2nb.novel.common.cache.CacheKey; +import com.java2nb.novel.common.cache.CacheService; +import com.java2nb.novel.common.utils.Constants; +import com.rabbitmq.client.Channel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Component; + + +/** + * @author 11797 + */ +@Component +@Slf4j +@RequiredArgsConstructor +public class BookVisitAddListener { + + private final BookService bookService; + + private final CacheService cacheService; + + + private final RedissonClient redissonClient; + + + + + /** + * 更新数据库 + * 流量削峰，每本小说累积10个点击更新一次 + */ + @SneakyThrows + @RabbitListener(queues = {"UPDATE-DB-QUEUE"}) + public void updateDb(Long bookId, Channel channel, Message message) { + + log.debug("收到更新数据库消息：" + bookId); + RLock lock = redissonClient.getLock("addVisitCountToDb"); + lock.lock(); + try { + Integer visitCount = (Integer) cacheService.getObject(CacheKey.BOOK_ADD_VISIT_COUNT + bookId); + if (visitCount == null) { + visitCount = 0; + } + cacheService.setObject(CacheKey.BOOK_ADD_VISIT_COUNT + bookId, ++visitCount); + if (visitCount >= Constants.ADD_MAX_VISIT_COUNT) { + bookService.addVisitCount(bookId, visitCount); + cacheService.del(CacheKey.BOOK_ADD_VISIT_COUNT + bookId); + } + }catch (Exception e){ + log.error("更新数据库失败"+bookId); + } + + lock.unlock(); + + Thread.sleep(1000 * 2); + + + + } + + + +} diff --git a/novel-book/book-service/src/main/resources/bootstrap.yml b/novel-book/book-service/src/main/resources/bootstrap.yml index 8fae4a1..c57a47c 100644 --- a/novel-book/book-service/src/main/resources/bootstrap.yml +++ b/novel-book/book-service/src/main/resources/bootstrap.yml @@ -3,3 +3,15 @@ spring: name: book-service profiles: active: dev + + cloud: + nacos: + config: + ext‐config[0]: + data‐id: novel-redis.yml + group: novel-common + refresh: true + ext‐config[1]: + data‐id: novel-rabbitmq.yml + group: novel-common + refresh: true \ No newline at end of file diff --git a/novel-common/pom.xml b/novel-common/pom.xml index 19efa4f..81102e8 100644 --- a/novel-common/pom.xml +++ b/novel-common/pom.xml @@ -48,6 +48,12 @@ spring-boot-starter-redis + + org.redisson + redisson-spring-boot-starter + ${redisson.version} + + mysql mysql-connector-java @@ -105,6 +111,11 @@ feign-httpclient + + org.springframework.boot + spring-boot-starter-amqp + + diff --git a/novel-common/src/main/java/com/java2nb/novel/common/config/RabbitConfig.java b/novel-common/src/main/java/com/java2nb/novel/common/config/RabbitConfig.java new file mode 100644 index 0000000..d66decd --- /dev/null +++ b/novel-common/src/main/java/com/java2nb/novel/common/config/RabbitConfig.java @@ -0,0 +1,62 @@ +package com.java2nb.novel.common.config; + +import org.springframework.amqp.core.Binding; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.FanoutExchange; +import org.springframework.amqp.core.Queue; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author 11797 + */ +@Configuration +@ConditionalOnProperty(prefix = "spring.rabbitmq", name = "host", matchIfMissing = false) +public class RabbitConfig { + + + /** + * 更新数据库队列 + */ + @Bean + public Queue updateDbQueue() { + return new Queue("UPDATE-DB-QUEUE", true); + } + + /** + * 更新数据库队列 + */ + @Bean + public Queue updateEsQueue() { + return new Queue("UPDATE-ES-QUEUE", true); + } + + + /** + * 增加点击量交换机 + */ + @Bean + public FanoutExchange addVisitExchange() { + return new FanoutExchange("ADD-BOOK-VISIT-EXCHANGE"); + } + + /** + * 更新搜索引擎队列绑定到增加点击量交换机中 + */ + @Bean + public Binding updateEsBinding() { + + return BindingBuilder.bind(updateEsQueue()).to(addVisitExchange()); + } + + /** + * 更新数据库绑定到增加点击量交换机中 + */ + @Bean + public Binding updateDbBinding() { + return BindingBuilder.bind(updateDbQueue()).to(addVisitExchange()); + } + + +} diff --git a/novel-common/src/main/resources/application-common.yml b/novel-common/src/main/resources/application-common.yml index 7ae9510..53c62fd 100644 --- a/novel-common/src/main/resources/application-common.yml +++ b/novel-common/src/main/resources/application-common.yml @@ -19,6 +19,12 @@ feign: httpclient: enabled: true +#关掉mq的健康检查，防止某些没有用到mq的服务启动报错，个别服务如需mq监控，单独开启 +management: + health: + rabbit: + enabled: false + diff --git a/novel-search/src/main/java/com/java2nb/novel/search/listener/BookVisitAddListener.java b/novel-search/src/main/java/com/java2nb/novel/search/listener/BookVisitAddListener.java new file mode 100644 index 0000000..e7b6136 --- /dev/null +++ b/novel-search/src/main/java/com/java2nb/novel/search/listener/BookVisitAddListener.java @@ -0,0 +1,67 @@ +package com.java2nb.novel.search.listener; + +import com.java2nb.novel.book.entity.Book; +import com.java2nb.novel.common.cache.CacheKey; +import com.java2nb.novel.common.cache.CacheService; +import com.java2nb.novel.search.feign.BookFeignClient; +import com.java2nb.novel.search.service.SearchService; +import com.rabbitmq.client.Channel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Component; + + +/** + * @author 11797 + */ +@Component +@Slf4j +@RequiredArgsConstructor +public class BookVisitAddListener { + + + private final CacheService cacheService; + + private final SearchService searchService; + + private final RedissonClient redissonClient; + + private final BookFeignClient bookFeignClient; + + + + + + /** + * 更新搜索引擎 + * 流量削峰，每本小说1个小时更新一次 + */ + @RabbitListener(queues = {"UPDATE-ES-QUEUE"}) + public void updateEs(Long bookId, Channel channel, Message message) { + + log.debug("收到更新搜索引擎消息：" + bookId); + RLock lock = redissonClient.getLock("addVisitCountToEs"); + lock.lock(); + if (cacheService.get(CacheKey.ES_IS_UPDATE_VISIT + bookId) == null) { + cacheService.set(CacheKey.ES_IS_UPDATE_VISIT + bookId, "1", 60 * 60); + try { + Thread.sleep(1000 * 5); + Book book = bookFeignClient.queryBookById(bookId); + searchService.importToEs(book); + }catch (Exception e){ + cacheService.del(CacheKey.ES_IS_UPDATE_VISIT + bookId); + log.error("更新搜索引擎失败"+bookId); + } + + } + lock.unlock(); + + + } + + +} diff --git a/novel-search/src/main/java/com/java2nb/novel/search/schedule/BookToEsSchedule.java b/novel-search/src/main/java/com/java2nb/novel/search/schedule/BookToEsSchedule.java index 8dde727..fb5314e 100644 --- a/novel-search/src/main/java/com/java2nb/novel/search/schedule/BookToEsSchedule.java +++ b/novel-search/src/main/java/com/java2nb/novel/search/schedule/BookToEsSchedule.java @@ -7,6 +7,8 @@ import com.java2nb.novel.search.feign.BookFeignClient; import com.java2nb.novel.search.service.SearchService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -16,6 +18,7 @@ import java.util.List; /** * 小说数据导入搜索引擎定时任务 + * * @author xiongxiaoyang * @version 1.0 * @since 2020/5/27 @@ -30,49 +33,47 @@ public class BookToEsSchedule { private final CacheService cacheService; - private final SearchService searchService; + private final RedissonClient redissonClient; + /** * 1分钟导入一次 */ @Scheduled(fixedRate = 1000 * 60) public void saveToEs() { - //TODO 引入Redisson框架实现分布式锁 - //可以重复更新，只是效率可能略有降低，所以暂不实现分布式锁 - if (cacheService.get(CacheKey.ES_TRANS_LOCK) == null) { - cacheService.set(CacheKey.ES_TRANS_LOCK, "1", 60 * 20); - try { - //查询需要更新的小说 - Date lastDate = (Date) cacheService.getObject(CacheKey.ES_LAST_UPDATE_TIME); - if (lastDate == null) { - lastDate = new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-01"); - } + RLock lock = redissonClient.getLock("saveToEs"); + lock.lock(); - - List books = bookFeignClient.queryBookByMinUpdateTime(lastDate, 100); - for (Book book : books) { - searchService.importToEs(book); - lastDate = book.getUpdateTime(); - Thread.sleep(5000); - - } - - - - cacheService.setObject(CacheKey.ES_LAST_UPDATE_TIME, lastDate); - - } catch (Exception e) { - log.error(e.getMessage(), e); + try { + //查询需要更新的小说 + Date lastDate = (Date) cacheService.getObject(CacheKey.ES_LAST_UPDATE_TIME); + if (lastDate == null) { + lastDate = new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-01"); } - cacheService.del(CacheKey.ES_TRANS_LOCK); + List books = bookFeignClient.queryBookByMinUpdateTime(lastDate, 100); + for (Book book : books) { + searchService.importToEs(book); + lastDate = book.getUpdateTime(); + Thread.sleep(5000); + + } + + + cacheService.setObject(CacheKey.ES_LAST_UPDATE_TIME, lastDate); + + } catch (Exception e) { + log.error(e.getMessage(), e); } + lock.unlock(); } + + } diff --git a/novel-search/src/main/resources/bootstrap.yml b/novel-search/src/main/resources/bootstrap.yml index 735b5e7..2c4cef2 100644 --- a/novel-search/src/main/resources/bootstrap.yml +++ b/novel-search/src/main/resources/bootstrap.yml @@ -9,4 +9,8 @@ spring: ext‐config[0]: data‐id: novel-redis.yml group: novel-common + refresh: true + ext‐config[1]: + data‐id: novel-rabbitmq.yml + group: novel-common refresh: true \ No newline at end of file