diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/core/constant/CacheKeyConstans.java b/novel-front/src/main/java/xyz/zinglizingli/books/core/constant/CacheKeyConstans.java index f76e594..3e45bb6 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/core/constant/CacheKeyConstans.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/core/constant/CacheKeyConstans.java @@ -7,4 +7,5 @@ public class CacheKeyConstans { public static final String EMAIL_URL_PREFIX_KEY = "emailUrlPrefixKey"; public static final String RANDOM_NEWS_CONTENT_KEY = "randomNewsContentKey"; public static final String REC_BOOK_LIST_KEY = "recBookListKey"; + public static final String BOOK_NUMBER_KEY= "bookNumberKey"; } diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java b/novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java index 184fd10..53b84f1 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java @@ -3,11 +3,13 @@ package xyz.zinglizingli.books.core.crawl; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import xyz.zinglizingli.books.core.utils.Constants; import xyz.zinglizingli.books.mapper.BookParseLogMapper; import xyz.zinglizingli.books.po.*; import xyz.zinglizingli.books.service.BookService; import xyz.zinglizingli.books.core.utils.CatUtil; +import xyz.zinglizingli.common.cache.CommonCacheUtil; import xyz.zinglizingli.common.utils.ExcutorUtils; import xyz.zinglizingli.common.utils.RestTemplateUtil; @@ -28,6 +30,12 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource { @Autowired private BookService bookService; + @Autowired + private CommonCacheUtil cacheUtil; + + @Value("${books.maxNum}") + private Integer maxNumBooks; + @Override public void parse() { @@ -168,8 +176,11 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource { //查询该书籍已存在目录号 Map hasIndexs = bookService.queryIndexByBookNameAndAuthor(bookName, author); + //查询数据库书籍数量 + long bookNumber = bookService.queryBookNumber(); + //更新和插入分别开，此处只做更新 - if (hasIndexs.size() > 0) { + if (hasIndexs.size() > 0 || bookNumber < maxNumBooks) { while (isFindIndex) { BookIndex hasIndex = hasIndexs.get(indexNum); String indexName = indexListMatch.group(2); diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java b/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java index 73b9f7e..ed0badd 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java @@ -517,4 +517,17 @@ public class BookService { bookParseLogMapper.deleteByPrimaryKey(id); } + + /** + * 查询数据库书籍数量 + * */ + public int queryBookNumber() { + + Integer bookNumber = (Integer) cacheUtil.getObject(CacheKeyConstans.BOOK_NUMBER_KEY); + if(bookNumber == null){ + bookNumber = bookMapper.countByExample(new BookExample()); + cacheUtil.setObject(CacheKeyConstans.BOOK_NUMBER_KEY,bookNumber,60*5); + } + return bookNumber; + } } diff --git a/novel-front/src/main/resources/application.yml b/novel-front/src/main/resources/application.yml index f5d249e..33f3033 100644 --- a/novel-front/src/main/resources/application.yml +++ b/novel-front/src/main/resources/application.yml @@ -83,6 +83,8 @@ books: lowestScore: 6.0 #小说的更新间隔（分） updatePeriod: 1 + #爬取小说数量 + maxNum: 300000 #爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说 ，4：百书斋 更多网站解析中，敬请期待 crawl: diff --git a/novel-front/src/main/resources/templates/books/index.html b/novel-front/src/main/resources/templates/books/index.html new file mode 100644 index 0000000..8ae3e72 --- /dev/null +++ b/novel-front/src/main/resources/templates/books/index.html @@ -0,0 +1,283 @@ + + + + + + + + 精品小说楼 + + + + + + + + + +
+ + + + + + + + + + + + + +
+
+ +
+ +
+
+ +
+
+ +
+
+
+

本站推荐

+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+
+
+
+
+

热门小说推荐

+
+ +
+
+
+ +
+ +
+
+
    +
  • +
  • +
  • +
+
+
+ +
+ + +
+
+ +
+ +
+
+ +
+

最近更新小说 +
更多 +
+

+ +
+ +
+
+
+ + +
- +
+
+ +
+
+ +
+ +
+ + +
+
+ +
+
+ +
+
+
+
+ + + + + + + + + + \ No newline at end of file