From be3cf1bb913f9c74befb1a04349f625c96c32ce8 Mon Sep 17 00:00:00 2001 From: xiongxiaoyang <773861846@qq.com> Date: Wed, 11 Dec 2019 14:23:30 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=BB=93=E6=9E=84=E8=B0=83?= =?UTF-8?q?=E6=95=B4=EF=BC=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E6=97=A0=E6=95=88?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=AE=9A=E6=97=B6=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../books/mapper/BookContentMapper.java | 5 +++ .../books/mapper/BookIndexMapper.java | 5 ++- .../zinglizingli/books/mapper/BookMapper.java | 6 +++- .../books/service/BookService.java | 21 ++++++++++-- .../books/service/UserService.java | 2 +- .../books/web/UserController.java | 4 +-- .../constant/CacheKeyConstans.java | 2 +- .../{books => common}/enums/PicSaveType.java | 2 +- .../schedule/ClearInvilidDataSchedule.java | 33 +++++++++++++++++++ .../common/schedule/CrawlBooksSchedule.java | 4 +-- .../util => common/utils}/ExcutorUtils.java | 2 +- .../{books/util => common/utils}/MD5Util.java | 2 +- .../utils}/RandomValueUtil.java | 2 +- .../util => common/utils}/UUIDUtils.java | 2 +- .../common/web/IndexController.java | 3 +- .../src/main/resources/application.yml | 4 +-- .../src/main/resources/logback-boot.xml | 2 +- .../mybatis/mapping/BookContentMapper.xml | 8 +++++ .../mybatis/mapping/BookIndexMapper.xml | 10 +++--- .../resources/mybatis/mapping/BookMapper.xml | 7 ++++ 20 files changed, 99 insertions(+), 27 deletions(-) rename novel-front/src/main/java/xyz/zinglizingli/{books => common}/constant/CacheKeyConstans.java (92%) rename novel-front/src/main/java/xyz/zinglizingli/{books => common}/enums/PicSaveType.java (90%) create mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/schedule/ClearInvilidDataSchedule.java rename novel-front/src/main/java/xyz/zinglizingli/{books/util => common/utils}/ExcutorUtils.java (93%) rename novel-front/src/main/java/xyz/zinglizingli/{books/util => common/utils}/MD5Util.java (96%) rename novel-front/src/main/java/xyz/zinglizingli/{books/util => common/utils}/RandomValueUtil.java (99%) rename novel-front/src/main/java/xyz/zinglizingli/{books/util => common/utils}/UUIDUtils.java (98%) diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookContentMapper.java b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookContentMapper.java index f389b28..f6ecef5 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookContentMapper.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookContentMapper.java @@ -30,4 +30,9 @@ public interface BookContentMapper { int updateByPrimaryKey(BookContent record); void insertBatch(List bookContent); + + /** + * 清除无效内容 + * */ + void clearInvilidContent(); } \ No newline at end of file diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookIndexMapper.java b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookIndexMapper.java index ac52e37..ff3d113 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookIndexMapper.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookIndexMapper.java @@ -33,7 +33,10 @@ public interface BookIndexMapper { void insertBatch(List bookIndex); - String queryNewstIndexName(@Param("bookId") Long bookId); + /** + * 清除无效章节 + * */ + void clearInvilidIndex(); } \ No newline at end of file diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java index 2bc0c5a..edee506 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java @@ -36,10 +36,14 @@ public interface BookMapper { void addVisitCount(@Param("bookId") Long bookId); - List queryEndBookIdList(); /** * 查询推荐书籍数据 * */ List queryRecBooks(List> configMap); + + /** + * 清理无效书籍(1个月前更新的无章节书籍) + * */ + void clearInvilidBook(); } \ No newline at end of file diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java b/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java index db2097b..586161c 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java @@ -15,11 +15,11 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import tk.mybatis.orderbyhelper.OrderByHelper; -import xyz.zinglizingli.books.constant.CacheKeyConstans; -import xyz.zinglizingli.books.enums.PicSaveType; +import xyz.zinglizingli.common.constant.CacheKeyConstans; +import xyz.zinglizingli.common.enums.PicSaveType; import xyz.zinglizingli.books.mapper.*; import xyz.zinglizingli.books.po.*; -import xyz.zinglizingli.books.util.UUIDUtils; +import xyz.zinglizingli.common.utils.UUIDUtils; import xyz.zinglizingli.common.cache.CommonCacheUtil; import xyz.zinglizingli.common.utils.RestTemplateUtil; @@ -536,4 +536,19 @@ public class BookService { public List queryRecBooks(List> configMap) { return bookMapper.queryRecBooks(configMap); } + + /** + * 清理数据库中无效数据 + * */ + public void clearInvilidData() { + + //清除无效内容 + bookContentMapper.clearInvilidContent(); + + //清除无效章节 + bookIndexMapper.clearInvilidIndex(); + + //清楚无效书籍 + bookMapper.clearInvilidBook(); + } } diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/service/UserService.java b/novel-front/src/main/java/xyz/zinglizingli/books/service/UserService.java index d151c72..5e80ad1 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/service/UserService.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/service/UserService.java @@ -9,7 +9,7 @@ import xyz.zinglizingli.books.po.User; import xyz.zinglizingli.books.po.UserExample; import xyz.zinglizingli.books.po.UserRefBook; import xyz.zinglizingli.books.po.UserRefBookExample; -import xyz.zinglizingli.books.util.MD5Util; +import xyz.zinglizingli.common.utils.MD5Util; import java.util.List; diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/web/UserController.java b/novel-front/src/main/java/xyz/zinglizingli/books/web/UserController.java index 0e8e006..f5f1c0a 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/web/UserController.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/web/UserController.java @@ -2,16 +2,14 @@ package xyz.zinglizingli.books.web; import lombok.RequiredArgsConstructor; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import xyz.zinglizingli.books.po.User; -import xyz.zinglizingli.books.service.BookService; import xyz.zinglizingli.books.service.UserService; -import xyz.zinglizingli.books.util.UUIDUtils; +import xyz.zinglizingli.common.utils.UUIDUtils; import xyz.zinglizingli.common.cache.CommonCacheUtil; import java.util.*; diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/constant/CacheKeyConstans.java b/novel-front/src/main/java/xyz/zinglizingli/common/constant/CacheKeyConstans.java similarity index 92% rename from novel-front/src/main/java/xyz/zinglizingli/books/constant/CacheKeyConstans.java rename to novel-front/src/main/java/xyz/zinglizingli/common/constant/CacheKeyConstans.java index 898daf7..ac2ccef 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/constant/CacheKeyConstans.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/constant/CacheKeyConstans.java @@ -1,4 +1,4 @@ -package xyz.zinglizingli.books.constant; +package xyz.zinglizingli.common.constant; public class CacheKeyConstans { public static final String HOT_BOOK_LIST_KEY = "hotBookListKey"; diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/enums/PicSaveType.java b/novel-front/src/main/java/xyz/zinglizingli/common/enums/PicSaveType.java similarity index 90% rename from novel-front/src/main/java/xyz/zinglizingli/books/enums/PicSaveType.java rename to novel-front/src/main/java/xyz/zinglizingli/common/enums/PicSaveType.java index 99fc1d0..3db8d34 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/enums/PicSaveType.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/enums/PicSaveType.java @@ -1,4 +1,4 @@ -package xyz.zinglizingli.books.enums; +package xyz.zinglizingli.common.enums; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/ClearInvilidDataSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/ClearInvilidDataSchedule.java new file mode 100644 index 0000000..79dddc5 --- /dev/null +++ b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/ClearInvilidDataSchedule.java @@ -0,0 +1,33 @@ +package xyz.zinglizingli.common.schedule; + + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import xyz.zinglizingli.books.service.BookService; + +/** + * 清理数据库中无效数据 + * + * @author 11797*/ +@Service +@RequiredArgsConstructor +@Slf4j +public class ClearInvilidDataSchedule { + + private final BookService bookService; + + /** + * 每天凌晨两点清理一次 + */ + @Scheduled(cron = "0 0 2 * * ?") + public void clear() { + + log.debug("ClearInvilidDataSchedule。。。。。。。。。。。。"); + + bookService.clearInvilidData(); + + + } +} diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java index e1bbb46..1d377af 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java @@ -13,7 +13,7 @@ import xyz.zinglizingli.books.po.Book; import xyz.zinglizingli.books.po.BookContent; import xyz.zinglizingli.books.po.BookIndex; import xyz.zinglizingli.books.service.BookService; -import xyz.zinglizingli.books.util.ExcutorUtils; +import xyz.zinglizingli.common.utils.ExcutorUtils; import xyz.zinglizingli.common.utils.RestTemplateUtil; import java.text.SimpleDateFormat; @@ -88,7 +88,6 @@ public class CrawlBooksSchedule { Pattern pattern = compile("value=\"(\\d+)/(\\d+)\""); Matcher matcher = pattern.matcher(forObject); boolean isFind = matcher.find(); - System.out.println("匹配分页数" + isFind); if (isFind) { //解析第一页书籍的数据 Pattern bookPatten = compile("href=\"/(\\d+_\\d+)/\""); @@ -283,7 +282,6 @@ public class CrawlBooksSchedule { Pattern pattern = compile("value=\"(\\d+)/(\\d+)\""); Matcher matcher = pattern.matcher(forObject); boolean isFind = matcher.find(); - System.out.println("匹配分页数" + isFind); if (isFind) { //解析第一页书籍的数据 Pattern bookPatten = compile("href=\"/(bqge\\d+)/\""); diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/util/ExcutorUtils.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/ExcutorUtils.java similarity index 93% rename from novel-front/src/main/java/xyz/zinglizingli/books/util/ExcutorUtils.java rename to novel-front/src/main/java/xyz/zinglizingli/common/utils/ExcutorUtils.java index 0f5dd70..f660cff 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/util/ExcutorUtils.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/utils/ExcutorUtils.java @@ -1,4 +1,4 @@ -package xyz.zinglizingli.books.util; +package xyz.zinglizingli.common.utils; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/util/MD5Util.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/MD5Util.java similarity index 96% rename from novel-front/src/main/java/xyz/zinglizingli/books/util/MD5Util.java rename to novel-front/src/main/java/xyz/zinglizingli/common/utils/MD5Util.java index cb0ea59..4209365 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/util/MD5Util.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/utils/MD5Util.java @@ -1,4 +1,4 @@ -package xyz.zinglizingli.books.util; +package xyz.zinglizingli.common.utils; import java.security.MessageDigest; diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/util/RandomValueUtil.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/RandomValueUtil.java similarity index 99% rename from novel-front/src/main/java/xyz/zinglizingli/books/util/RandomValueUtil.java rename to novel-front/src/main/java/xyz/zinglizingli/common/utils/RandomValueUtil.java index ce5ccc6..d7a6524 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/util/RandomValueUtil.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/utils/RandomValueUtil.java @@ -1,4 +1,4 @@ -package xyz.zinglizingli.books.util; +package xyz.zinglizingli.common.utils; diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/util/UUIDUtils.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/UUIDUtils.java similarity index 98% rename from novel-front/src/main/java/xyz/zinglizingli/books/util/UUIDUtils.java rename to novel-front/src/main/java/xyz/zinglizingli/common/utils/UUIDUtils.java index 103fcd9..8ed8398 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/util/UUIDUtils.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/utils/UUIDUtils.java @@ -1,4 +1,4 @@ -package xyz.zinglizingli.books.util; +package xyz.zinglizingli.common.utils; import java.util.UUID; diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/web/IndexController.java b/novel-front/src/main/java/xyz/zinglizingli/common/web/IndexController.java index 473f554..c04a418 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/web/IndexController.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/web/IndexController.java @@ -2,11 +2,10 @@ package xyz.zinglizingli.common.web; import lombok.RequiredArgsConstructor; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.RequestMapping; -import xyz.zinglizingli.books.constant.CacheKeyConstans; +import xyz.zinglizingli.common.constant.CacheKeyConstans; import xyz.zinglizingli.books.po.Book; import xyz.zinglizingli.books.service.BookService; import xyz.zinglizingli.common.cache.CommonCacheUtil; diff --git a/novel-front/src/main/resources/application.yml b/novel-front/src/main/resources/application.yml index 46026bd..dd5fe56 100644 --- a/novel-front/src/main/resources/application.yml +++ b/novel-front/src/main/resources/application.yml @@ -3,9 +3,9 @@ server: spring: datasource: - url: jdbc:mysql://47.106.243.172:3306/books?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai + url: jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai username: books - password: books!8888 + password: books # url: jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai # username: root # password: test123456 diff --git a/novel-front/src/main/resources/logback-boot.xml b/novel-front/src/main/resources/logback-boot.xml index 7f02933..b1fcc5d 100644 --- a/novel-front/src/main/resources/logback-boot.xml +++ b/novel-front/src/main/resources/logback-boot.xml @@ -56,7 +56,7 @@ - + \ No newline at end of file diff --git a/novel-front/src/main/resources/mybatis/mapping/BookContentMapper.xml b/novel-front/src/main/resources/mybatis/mapping/BookContentMapper.xml index 1eb1d04..b29044f 100644 --- a/novel-front/src/main/resources/mybatis/mapping/BookContentMapper.xml +++ b/novel-front/src/main/resources/mybatis/mapping/BookContentMapper.xml @@ -225,4 +225,12 @@ + + + delete t1 from book_content t1 left join book_index t2 + on t1.book_id = t2.book_id and t1.index_num = t2.index_num + left join book t3 on t1.book_id = t3.id + where t2.id is null or t3.id is null + + \ No newline at end of file diff --git a/novel-front/src/main/resources/mybatis/mapping/BookIndexMapper.xml b/novel-front/src/main/resources/mybatis/mapping/BookIndexMapper.xml index 5400ea4..72cdfd0 100644 --- a/novel-front/src/main/resources/mybatis/mapping/BookIndexMapper.xml +++ b/novel-front/src/main/resources/mybatis/mapping/BookIndexMapper.xml @@ -213,11 +213,13 @@ - + + delete t1 from book_index t1 left join book_content t2 + on t1.book_id = t2.book_id and t1.index_num = t2.index_num + left join book t3 on t1.book_id = t3.id + where t2.id is null or t3.id is null + \ No newline at end of file diff --git a/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml b/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml index 909807c..7b9fa6f 100644 --- a/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml +++ b/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml @@ -388,4 +388,11 @@ limit 3 + + + + delete from book + where id not in (select book_id from book_index) + and update_time DATE_SUB(curdate(),INTERVAL 10 DAY) + \ No newline at end of file