2.6.0发布，新增笔趣窝源

README.md

@@ -1,12 +1,34 @@
-[![index](./assets/热门云产品1040.100.jpg)](https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console)
+[![index]( https://s1.ax1x.com/2020/06/22/NG6N2n.png )](https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console)
 
 # 小说精品屋
 
-#### 项目文档
+#### 新项目：小说精品屋-plus（推荐）
 
-通用文档：[点击前往]( http://www.java2nb.com/article/5.html )
+小说精品屋-plus致力于打造一个完整的可商用、可学习的小说门户平台。小说精品屋-plus是在小说精品屋的基础上，重新进行了数据库设计、代码重构和功能增强，提升了程序整体的可读性和性能，增加了很多商用特性。 
 
- 非JAVA开发参考安装文档：[点击前往]( https://my.oschina.net/java2nb/blog/3145593 )
+Gitee仓库地址： https://gitee.com/xiongxyang/novel-plus
+
+GitHub仓库地址： https://github.com/201206030/novel-plus
+
+#### 新项目：小说精品屋-微服务版（推荐）
+
+基于小说精品屋-plus构建的Spring Cloud 微服务小说门户平台，可用于学习和商用。
+
+Gitee仓库地址： https://gitee.com/xiongxyang/novel-cloud
+
+GitHub仓库地址： https://github.com/201206030/novel-cloud
+
+#### 前言
+
+安装前请先阅读完此文档，了解项目基础配置和模块功能，再根据安装文档安装项目，避免一些不必要的错误。
+
+#### 安装文档
+
+源码安装文档（适合有一定技术基础的人）：[点击前往](https://my.oschina.net/java2nb/blog/3145593)  
+
+包安装文档（适合非技术人员）：[点击前往](https://my.oschina.net/java2nb/blog/3146627)  
+
+宝塔安装教程：[点击前往](https://www.daniao.org/7822.html )
 
 #### 项目介绍
 
@@ -105,7 +127,7 @@ novel-admin ：平台后台管理系统源码（独立项目，按需安装）
 
 ![mini4](./assets/android_index.png)
 
-#### 安装教程
+#### 安装说明
 
 数据库安装：
 
@@ -119,7 +141,7 @@ novel-admin ：平台后台管理系统源码（独立项目，按需安装）
 1.  运行script/crawlbook/crawlbook.bat脚本文件。（适用于本地多机器运行） 
 2. 安装后台管理系统后，打开爬虫管理菜单，点击爬虫运行按钮。（适用于线上环境运行，会占用较多服务器资源）
 
-平台后台管理系统安装（独立项目，按需安装） ：
+平台后台管理系统安装（独立项目，按需安装）（已停止维护，爬虫功能请使用crawlbook.bat） ：
 
 1. 修改application.yml文件中数据库配置。
 
@@ -225,4 +247,4 @@ novel-admin ：平台后台管理系统源码（独立项目，按需安装）
 
 精品小说屋所有相关项目均已在开源中国公开，感兴趣的可进入[开源中国](https://www.oschina.net/p/fiction_house)按关键字`精品小说屋`搜索。
 
-[![index](./assets/120060.jpg)](https://www.aliyun.com/minisite/goods?userCode=uf4nasee )
+[![index]( https://s1.ax1x.com/2020/06/22/NG6N2n.png)](https://www.aliyun.com/minisite/goods?userCode=uf4nasee )

novel-front/pom.xml

@@ -10,7 +10,7 @@
     </parent>
     <groupId>xyz.zinglizingli</groupId>
     <artifactId>novel-front</artifactId>
-    <version>2.3.0.beta</version>
+    <version>2.5.0.beta</version>
     <name>novel-front</name>
     <description>小说精品楼-前台web网站</description>
 

novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBiquwoConfig.java

@@ -0,0 +1,27 @@
+package xyz.zinglizingli.books.core.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import xyz.zinglizingli.books.core.crawl.BaseHtmlCrawlSource;
+import xyz.zinglizingli.books.core.crawl.BiquCrawlSource;
+
+/**
+ * @author 11797
+ */
+@Slf4j
+@Configuration
+public class CrawlBiquwoConfig {
+
+
+    @Bean
+    @ConfigurationProperties(prefix = "biquwo.crawlsource") // prefix值必须是application.yml中对应属性的前缀
+    @ConditionalOnProperty(prefix = "crawl.website",name = "type",havingValue = "6")
+    public BaseHtmlCrawlSource biquwoCrawlSource() {
+        return new BiquCrawlSource();
+    }
+
+
+}

novel-front/src/main/java/xyz/zinglizingli/books/core/listener/StartListener.java

@@ -66,6 +66,7 @@ public class StartListener implements ServletContextListener {
 
                         log.info("updateBooks执行中。。。。。。。。。。。。");
                         crawlSource.update();
+                        Thread.sleep(2000);
                     } catch (Exception e) {
                         log.error(e.getMessage(), e);
                     }

novel-front/src/main/java/xyz/zinglizingli/books/core/schedule/Network2LocalPicSchedule.java

@@ -42,13 +42,16 @@ public class Network2LocalPicSchedule {
 
         log.info("Network2LocalPicSchedule。。。。。。。。。。。。");
 
-
         Integer offset = 0, limit = 100;
         List<Book> networkPicBooks;
         do {
             networkPicBooks = bookService.queryNetworkPicBooks(limit, offset);
             for (Book book : networkPicBooks) {
-                bookService.updateBook(book, book.getId());
+                try {
+                    bookService.networkPicToLocal(book);
+                } catch (Exception e) {
+                    log.error(e.getMessage(), e);
+                }
             }
             offset += limit;
         } while (networkPicBooks.size() > 0);

novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java

@@ -54,11 +54,10 @@ public class BookService {
     private String picSavePath;
 
 
-
     /**
      * 保存章节目录和内容
-     * */
-    public void saveBookAndIndexAndContent(Book book, List<BookIndex> bookIndex, List<BookContent> bookContent){
+     */
+    public void saveBookAndIndexAndContent(Book book, List<BookIndex> bookIndex, List<BookContent> bookContent) {
         //解决内部调用事物不生效的问题
         BookService bookService = SpringUtil.getBean(BookService.class);
 
@@ -72,7 +71,7 @@ public class BookService {
         if (books.size() > 0) {
             //更新
             bookId = books.get(0).getId();
-            if(picSaveType == PicSaveType.LOCAL.getValue() && books.get(0).getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)){
+            if (picSaveType == PicSaveType.LOCAL.getValue() && books.get(0).getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
                 book.setPicUrl(null);
             }
             updateBook(book, bookId);
@@ -126,7 +125,6 @@ public class BookService {
             }
 
 
-
         }
 
 
@@ -134,31 +132,45 @@ public class BookService {
 
     /**
      * 更新书籍
-     * */
+     */
     public void updateBook(Book book, Long bookId) {
         book.setId(bookId);
         String picSrc = book.getPicUrl();
-        if(picSaveType == PicSaveType.LOCAL.getValue() && StringUtils.isNotBlank(picSrc)){
+        if (picSaveType == PicSaveType.LOCAL.getValue() && StringUtils.isNotBlank(picSrc)) {
             try {
-                picSrc = FileUtil.network2Local(picSrc,picSavePath);
+                picSrc = FileUtil.network2Local(picSrc, picSavePath);
                 book.setPicUrl(picSrc);
-            }catch (Exception e){
-                log.error(e.getMessage(),e);
+            } catch (Exception e) {
+                log.error(e.getMessage(), e);
             }
 
         }
         bookMapper.updateByPrimaryKeySelective(book);
     }
 
+    /**
+     * 网络图片转本地
+     *
+     * @param book
+     */
+    public void networkPicToLocal(Book book) {
+        try {
+            book.setPicUrl(FileUtil.network2Local(book.getPicUrl(), picSavePath));
+            bookMapper.updateByPrimaryKeySelective(book);
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+
+    }
 
 
     /**
      * 批量插入章节目录表和章节内容表（自动修复错误章节）
-     * */
+     */
     @Transactional(rollbackFor = Exception.class)
     public void insertIndexListAndContentList(List<BookIndex> newBookIndexList, List<BookContent> newContentList) {
         long start = System.currentTimeMillis();
-        if(newBookIndexList.size() > 0) {
+        if (newBookIndexList.size() > 0) {
             //删除已存在的错误章节
             List<Integer> indexNumberList = newBookIndexList.stream().map(BookIndex::getIndexNum).collect(Collectors.toList());
             Long bookId = newBookIndexList.get(0).getBookId();
@@ -173,15 +185,15 @@ public class BookService {
             bookIndexMapper.insertBatch(newBookIndexList);
             bookContentMapper.insertBatch(newContentList);
         }
-        log.info("更新章节耗时："+(System.currentTimeMillis()-start));
+        log.info("更新章节耗时：" + (System.currentTimeMillis() - start));
     }
 
 
     /**
      * 生成随机访问次数
-     * */
+     */
     private Long generateVisitCount(Float score) {
-        int baseNum = (int)(score * 100);
+        int baseNum = (int) (score * 100);
         return Long.parseLong(baseNum + new Random().nextInt(1000) + "");
     }
 
@@ -206,10 +218,9 @@ public class BookService {
     }
 
 
-
     /**
      * 查询书籍的基础数据
-     * */
+     */
     public Book queryBaseInfo(Long bookId) {
 
         return bookMapper.selectByPrimaryKey(bookId);
@@ -217,7 +228,7 @@ public class BookService {
 
     /**
      * 查询最新更新的书籍列表
-     * */
+     */
     public List<BookIndex> queryNewIndexList(Long bookId) {
         PageHelper.startPage(1, 15);
         BookIndexExample example = new BookIndexExample();
@@ -229,7 +240,7 @@ public class BookService {
 
     /**
      * 查询书籍目录列表
-     * */
+     */
     public List<BookIndex> queryAllIndexList(Long bookId) {
         BookIndexExample example = new BookIndexExample();
         example.createCriteria().andBookIdEqualTo(bookId);
@@ -239,7 +250,7 @@ public class BookService {
 
     /**
      * 查询书籍章节内容
-     * */
+     */
     public BookContent queryBookContent(Long bookId, Integer indexNum) {
         BookContent content = (BookContent) cacheUtil.getObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum);
         if (content == null) {
@@ -256,12 +267,12 @@ public class BookService {
 
     /**
      * 增加访问次数
-     * */
+     */
     public void addVisitCount(Long bookId, String userId, Integer indexNum) {
 
         bookMapper.addVisitCount(bookId);
 
-        if(org.apache.commons.lang3.StringUtils.isNotBlank(userId)) {
+        if (org.apache.commons.lang3.StringUtils.isNotBlank(userId)) {
             userRefBookMapper.updateNewstIndex(bookId, userId, indexNum);
         }
 
@@ -269,13 +280,13 @@ public class BookService {
 
     /**
      * 查询章节名
-     * */
+     */
     public String queryIndexNameByBookIdAndIndexNum(Long bookId, Integer indexNum) {
 
         BookIndexExample example = new BookIndexExample();
         example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(indexNum);
         List<BookIndex> indexList = bookIndexMapper.selectByExample(example);
-        if(indexList != null && indexList.size() > 0 ) {
+        if (indexList != null && indexList.size() > 0) {
             return indexList.get(0).getIndexName();
         }
         return null;
@@ -283,7 +294,7 @@ public class BookService {
 
     /**
      * 查询最大和最小章节号
-     * */
+     */
     public List<Integer> queryMaxAndMinIndexNum(Long bookId) {
         List<Integer> result = new ArrayList<>();
         BookIndexExample example = new BookIndexExample();
@@ -300,7 +311,7 @@ public class BookService {
     /**
      * 查询该书籍已存在目录号
      */
-    public Map<Integer,BookIndex> queryIndexByBookNameAndAuthor(String bookName, String author) {
+    public Map<Integer, BookIndex> queryIndexByBookNameAndAuthor(String bookName, String author) {
         BookExample example = new BookExample();
         example.createCriteria().andBookNameEqualTo(bookName).andAuthorEqualTo(author);
         List<Book> books = bookMapper.selectByExample(example);
@@ -310,7 +321,7 @@ public class BookService {
             BookIndexExample bookIndexExample = new BookIndexExample();
             bookIndexExample.createCriteria().andBookIdEqualTo(bookId);
             List<BookIndex> bookIndices = bookIndexMapper.selectByExample(bookIndexExample);
-            if(bookIndices.size() > 0) {
+            if (bookIndices.size() > 0) {
                 return bookIndices.stream().collect(Collectors.toMap(BookIndex::getIndexNum, Function.identity()));
             }
 
@@ -321,11 +332,9 @@ public class BookService {
     }
 
 
-
-
     /**
      * 保存弹幕
-     * */
+     */
     public void sendBullet(Long contentId, String bullet) {
 
         ScreenBullet screenBullet = new ScreenBullet();
@@ -338,7 +347,7 @@ public class BookService {
 
     /**
      * 查询弹幕
-     * */
+     */
     public List<ScreenBullet> queryBullet(Long contentId) {
 
         ScreenBulletExample example = new ScreenBulletExample();
@@ -351,7 +360,7 @@ public class BookService {
 
     /**
      * 查询章节内容
-     * */
+     */
     public String queryContentList(Long bookId, int count) {
         BookContentExample example = new BookContentExample();
         example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(count);
@@ -360,7 +369,7 @@ public class BookService {
 
     /**
      * 查询章节数
-     * */
+     */
     public int countIndex(Long bookId) {
         BookIndexExample example = new BookIndexExample();
         example.createCriteria().andBookIdEqualTo(bookId);
@@ -368,11 +377,9 @@ public class BookService {
     }
 
 
-
-
     /**
      * 查询前一章节和后一章节号
-     * */
+     */
     public List<Integer> queryPreAndNextIndexNum(Long bookId, Integer indexNum) {
         List<Integer> result = new ArrayList<>();
         BookIndexExample example = new BookIndexExample();
@@ -399,14 +406,14 @@ public class BookService {
 
     /**
      * 查询推荐书籍数据
-     * */
+     */
     public List<Book> queryRecBooks(List<Map<String, String>> configMap) {
         return bookMapper.queryRecBooks(configMap);
     }
 
     /**
      * 清理数据库中无效数据
-     * */
+     */
     public void clearInvilidData() {
 
         //清除无效内容
@@ -423,27 +430,28 @@ public class BookService {
      * 查询网络图片的小说
      *
      * @param limit
-     * @param offset*/
+     * @param offset
+     */
     public List<Book> queryNetworkPicBooks(Integer limit, Integer offset) {
-        return bookMapper.queryNetworkPicBooks(limit,offset);
+        return bookMapper.queryNetworkPicBooks(limit, offset);
     }
 
     /**
      * 通过图片名查询小说数量
-     * */
+     */
     public int countByPicName(String fileName) {
         BookExample bookExample = new BookExample();
-        bookExample.createCriteria().andPicUrlLike('%'+fileName+'%');
+        bookExample.createCriteria().andPicUrlLike('%' + fileName + '%');
         return bookMapper.countByExample(bookExample);
     }
 
     /**
      * 添加解析日志
-     * */
+     */
     public void addBookParseLog(String bookUrl, String bookName, Float score, Byte priority) {
         BookParseLogExample example = new BookParseLogExample();
-        example.createCriteria().andBookUrlEqualTo(bookUrl).andCreateTimeGreaterThan(new Date(System.currentTimeMillis()-1000*60*60));
-        if(bookParseLogMapper.countByExample(example)==0) {
+        example.createCriteria().andBookUrlEqualTo(bookUrl).andCreateTimeGreaterThan(new Date(System.currentTimeMillis() - 1000 * 60 * 60));
+        if (bookParseLogMapper.countByExample(example) == 0) {
             BookParseLog bookParseLog = new BookParseLog();
             bookParseLog.setBookUrl(bookUrl);
             bookParseLog.setBookName(bookName);
@@ -456,16 +464,18 @@ public class BookService {
 
     /**
      * 查询解析日志
-     * */
+     */
     public List<BookParseLog> queryBookParseLogs() {
         List<BookParseLog> logs = bookParseLogMapper.queryBookParseLogs();
-        SpringUtil.getBean(BookService.class).addBookUpdateCount(logs);
+        if(logs.size()>0) {
+            SpringUtil.getBean(BookService.class).addBookUpdateCount(logs);
+        }
         return logs;
     }
 
     /**
      * 增加小说更新次数
-     * */
+     */
     @Async
     public void addBookUpdateCount(List<BookParseLog> logs) {
         bookParseLogMapper.addBookUpdateCount(logs);
@@ -473,9 +483,9 @@ public class BookService {
 
     /**
      * 删除已经成功更新的解析日志
-     * */
+     */
     public void deleteBookParseLogs(List<Long> successLogIds) {
-        if(successLogIds.size()>0) {
+        if (successLogIds.size() > 0) {
             BookParseLogExample example = new BookParseLogExample();
             example.createCriteria().andIdIn(successLogIds);
             bookParseLogMapper.deleteByExample(example);
@@ -484,35 +494,35 @@ public class BookService {
 
     /**
      * 查询书籍是否存在
-     * */
+     */
     public Boolean hasBook(String bookName, String author) {
         BookExample example = new BookExample();
         example.createCriteria().andBookNameEqualTo(bookName).andAuthorEqualTo(author);
-        return bookMapper.countByExample(example)>0;
+        return bookMapper.countByExample(example) > 0;
     }
 
     /**
      * 查询分类更新时间映射信息
-     * */
+     */
     public Map<Integer, Date> queryLastUpdateTime() {
         List<BookUpdateTimeLog> list = bookUpdateTimeLogMapper.selectByExample(new BookUpdateTimeLogExample());
 
-        return list.stream().collect(Collectors.toMap(BookUpdateTimeLog::getBookCatId, BookUpdateTimeLog::getLastUpdateTime,(key1, key2) -> key2));
+        return list.stream().collect(Collectors.toMap(BookUpdateTimeLog::getBookCatId, BookUpdateTimeLog::getLastUpdateTime, (key1, key2) -> key2));
 
     }
 
     /**
      * 更新分类时间日志
-     * */
+     */
     public void updateBookUpdateTimeLog(Map<Integer, Date> cat2Date) {
-        if(cat2Date.size()>0) {
+        if (cat2Date.size() > 0) {
             Set<Map.Entry<Integer, Date>> entries = cat2Date.entrySet();
-            for(Map.Entry<Integer, Date> entry : entries){
+            for (Map.Entry<Integer, Date> entry : entries) {
                 BookUpdateTimeLogExample example = new BookUpdateTimeLogExample();
                 example.createCriteria().andBookCatIdEqualTo(entry.getKey());
                 BookUpdateTimeLog entity = new BookUpdateTimeLog();
                 entity.setLastUpdateTime(entry.getValue());
-                bookUpdateTimeLogMapper.updateByExampleSelective(entity,example);
+                bookUpdateTimeLogMapper.updateByExampleSelective(entity, example);
 
             }
         }
@@ -520,7 +530,7 @@ public class BookService {
 
     /**
      * 删除已经成功更新的解析日志
-     * */
+     */
     public void deleteBookParseLog(Long id) {
         bookParseLogMapper.deleteByPrimaryKey(id);
 
@@ -528,13 +538,13 @@ public class BookService {
 
     /**
      * 查询数据库书籍数量
-     * */
+     */
     public int queryBookNumber() {
 
         Integer bookNumber = (Integer) cacheUtil.getObject(CacheKeyConstans.BOOK_NUMBER_KEY);
-        if(bookNumber == null){
+        if (bookNumber == null) {
             bookNumber = bookMapper.countByExample(new BookExample());
-            cacheUtil.setObject(CacheKeyConstans.BOOK_NUMBER_KEY,bookNumber,60*5);
+            cacheUtil.setObject(CacheKeyConstans.BOOK_NUMBER_KEY, bookNumber, 60 * 5);
         }
         return bookNumber;
     }

novel-front/src/main/resources/application-crawl.yml

@@ -86,3 +86,20 @@ biquge:
     intro-pattern: class="review">([^/]+)</p>
     catalog-url-pattern: <a\s+href="(/ddk\d+/all.html)">查看完整目录</a>
     catalog-pattern: <dd>\s*<a\s+href="(\d+\.html)"\s+title="([^"]+)">([^<]+)</a>\s*</dd>
+
+
+biquwo:
+  crawlsource:
+    index-url: http://m.biquwo.net
+    list-page-url: http://m.biquwo.net/sort{0}/0/{1}.html
+    book-url-pattern: href="/(dudu/\d+/\d+)/"
+    score-pattern: <div\s+class="score">(\d+\.\d+)分</div>
+    book-name-pattern: <p class="title">([^/]+)</p>
+    author-pattern: 作者：([^/]+)<
+    status-pattern: 状态：([^/]+)</li>
+    cat-pattern: 类别：([^/]+)</li>
+    update-time-pattern: 更新：(\d+-\d+-\d+\s\d+:\d+:\d+)</a>
+    pic-pattern: <img src="([^>]+)"\s+onerror="this.src=
+    intro-pattern: class="review">([^<]+)</p>
+    catalog-url-pattern: <a\s+href="(/dudu/\d+/\d+/all\.html)">查看完整目录</a>
+    catalog-pattern: <a\s+style=""\s+href="(/dudu/\d+/\d+/\d+\.html)">([^/]+)</a>

novel-front/src/main/resources/application.yml

@@ -86,17 +86,17 @@ books:
   #爬取小说数量
   maxNum: 300000
 
-#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说 ，4：百书斋 更多网站解析中，敬请期待
+#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说 ，4：百书斋,6: 笔趣窝 更多网站解析中，敬请期待
 crawl:
   website:
-    type: 4
+    type: 6
 
 
 
 
 pic:
   save:
-    type: 1 #图片保存方式, 1不保存，使用网络图片 ，2本地保存
+    type: 2 #图片保存方式, 1不保存，使用网络图片 ，2本地保存
     path: /var/pic  #图片保存路径
 
 

script/crawlbook/application.yml

@@ -9,7 +9,8 @@ mybatis:
 mysql: {charset: utf8mb4}
 books: {lowestScore: 6.0}
 crawl:
-  website: {type: '4'}
-  soft-novel: 0
-  manhua: 0
+  patten: '1'
+  website: {type: 6}
+  soft-novel: '0'
+  manhua: '0'
 logging: {config: 'classpath:logback-boot.xml'}