From 4d5cd12b92be6bc406ff2425041dcf25ed45eb5e Mon Sep 17 00:00:00 2001 From: xxy <773861846@qq.com> Date: Sat, 2 May 2020 20:05:43 +0800 Subject: [PATCH] =?UTF-8?q?bug=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java2nb/novel/core/cache/CacheKey.java | 4 ---- .../java2nb/novel/core/crawl/CrawlParser.java | 1 - .../java2nb/novel/service/BookService.java | 15 +++++++++++---- .../novel/service/impl/BookServiceImpl.java | 19 ++++++++++++++++++- .../novel/service/impl/CrawlServiceImpl.java | 7 +++---- 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java b/novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java index 2ae7149..4f3ee6d 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java @@ -46,8 +46,4 @@ public interface CacheKey { * */ String RUNNING_CRAWL_THREAD_KEY_PREFIX = "runningCrawlTreadDataKeyPrefix"; - /** - * 标记小说是否正在入库 - * */ - String NEW_BOOK_IN_SAVE = "newBookInSave"; } \ No newline at end of file diff --git a/novel-common/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java b/novel-common/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java index fa3c3e6..6f3d5e9 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java @@ -26,7 +26,6 @@ public class CrawlParser { public static final Integer BOOK_INDEX_LIST_KEY = 1; public static final Integer BOOK_CONTENT_LIST_KEY = 2; - private static final Integer BOOK_KEY = 3; @SneakyThrows public static Book parseBook(RuleBean ruleBean, String bookId) { diff --git a/novel-crawl/src/main/java/com/java2nb/novel/service/BookService.java b/novel-crawl/src/main/java/com/java2nb/novel/service/BookService.java index 925a108..a606b34 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/service/BookService.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/service/BookService.java @@ -5,7 +5,6 @@ import com.java2nb.novel.entity.BookContent; import com.java2nb.novel.entity.BookIndex; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -25,10 +24,10 @@ public interface BookService { /** * 更新书籍的爬虫属性 + * @param id 本站小说ID * @param sourceId 爬虫源ID - * @param bookId 源站小说ID - * */ - void updateCrawlProperties(Integer sourceId, String bookId); + * @param bookId 源站小说ID */ + void updateCrawlProperties(Long id, Integer sourceId, String bookId); /** * 通过分类ID查询分类名 @@ -74,4 +73,12 @@ public interface BookService { * @param bookId 小说ID * */ void updateCrawlLastTime(Long bookId); + + /** + * 通过小说名和作者名查询已存在的书籍 + * @param bookName 小说名 + * @param authorName 作者名 + * @return 小说对象 + * */ + Book queryBookByBookNameAndAuthorName(String bookName, String authorName); } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java index cd25629..dd99165 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java @@ -51,12 +51,13 @@ public class BookServiceImpl implements BookService { } @Override - public void updateCrawlProperties(Integer sourceId, String bookId) { + public void updateCrawlProperties(Long id, Integer sourceId, String bookId) { bookMapper.update(update(BookDynamicSqlSupport.book) .set(crawlSourceId) .equalTo(sourceId) .set(crawlBookId) .equalTo(bookId) + .where(BookDynamicSqlSupport.id,isEqualTo(id)) .build() .render(RenderingStrategies.MYBATIS3)); } @@ -176,6 +177,22 @@ public class BookServiceImpl implements BookService { bookMapper.updateByPrimaryKeySelective(book); } + @Override + public Book queryBookByBookNameAndAuthorName(String bookName, String authorName) { + List books = bookMapper.selectMany(select(BookDynamicSqlSupport.id).from(BookDynamicSqlSupport.book) + .where(BookDynamicSqlSupport.bookName, isEqualTo(bookName)) + .and(BookDynamicSqlSupport.authorName, isEqualTo(authorName)) + .build() + .render(RenderingStrategies.MYBATIS3)); + + if(books.size()>0){ + return books.get(0); + } + + return null; + + } + /** * 查询最后的章节 * */ diff --git a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java index ce66fdd..1c6e0b3 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java @@ -168,11 +168,10 @@ public class CrawlServiceImpl implements CrawlService { String bookId = bookIdMatcher.group(1); Book book = CrawlParser.parseBook(ruleBean, bookId); //这里只做新书入库，查询是否存在这本书 - boolean isExist = bookService.queryIsExistByBookNameAndAuthorName(book.getBookName(), book.getAuthorName()); + Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName()); //如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库 - if (!isExist && StringUtils.isBlank(cacheService.get(CacheKey.NEW_BOOK_IN_SAVE + book.getBookName() + "-" + book.getAuthorName()))) { + if (existBook == null) { //没有该书，可以入库 - cacheService.set(CacheKey.NEW_BOOK_IN_SAVE + book.getBookName() + "-" + book.getAuthorName(), "true", 60 * 30); book.setCatId(catId); //根据分类ID查询分类 book.setCatName(bookService.queryCatNameByCatId(catId)); @@ -194,7 +193,7 @@ public class CrawlServiceImpl implements CrawlService { } else { //只更新书籍的爬虫相关字段 - bookService.updateCrawlProperties(sourceId, bookId); + bookService.updateCrawlProperties(existBook.getId(),sourceId, bookId); } }catch (Exception e){ log.error(e.getMessage(),e);