优化章节字数算法，优化爬虫代码

novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java

@@ -1,9 +1,6 @@
 package com.java2nb.novel.core.crawl;
 
-import com.java2nb.novel.core.utils.HttpUtil;
+import com.java2nb.novel.core.utils.*;
-import com.java2nb.novel.core.utils.IdWorker;
-import com.java2nb.novel.core.utils.RandomBookInfoUtil;
-import com.java2nb.novel.core.utils.RestTemplateUtil;
 import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.entity.BookContent;
 import com.java2nb.novel.entity.BookIndex;
@@ -223,52 +220,40 @@ public class CrawlParser {
                     if (contentHtml != null && !contentHtml.contains("正在手打中")) {
                         String content = contentHtml.substring(contentHtml.indexOf(ruleBean.getContentStart()) + ruleBean.getContentStart().length());
                         content = content.substring(0, content.indexOf(ruleBean.getContentEnd()));
-                        //TODO插入章节目录和章节内容
+                        //插入章节目录和章节内容
                         BookIndex bookIndex = new BookIndex();
-
                         bookIndex.setIndexName(indexName);
                         bookIndex.setIndexNum(indexNum);
+                        Integer wordCount = StringUtil.getStrValidWordCount(content);
+                        bookIndex.setWordCount(wordCount);
                         indexList.add(bookIndex);
-                        BookContent bookContent = new BookContent();
 
+                        BookContent bookContent = new BookContent();
                         bookContent.setContent(content);
                         contentList.add(bookContent);
 
-                        //判断是新增还是更新
+
-                        if(hasIndexs.size() == 0){
+                        if (hasIndex != null) {
-                            //新书入库
+                            //章节更新
+                            bookIndex.setId(hasIndex.getId());
+                            bookContent.setIndexId(hasIndex.getId());
+                        } else {
+                            //章节插入
                             //设置目录和章节内容
                             Long indexId = idWorker.nextId();
                             lastIndexId = indexId;
                             lastIndexName = indexName;
                             bookIndex.setId(indexId);
                             bookIndex.setBookId(book.getId());
-                            Integer wordCount = bookContent.getContent().length();
+
-                            totalWordCount += wordCount;
-                            bookIndex.setWordCount(wordCount);
                             bookIndex.setCreateTime(currentDate);
-                            bookIndex.setUpdateTime(currentDate);
 
                             bookContent.setIndexId(indexId);
-
-                            //设置小说基础信息
-                            book.setWordCount(totalWordCount);
-                            book.setLastIndexId(lastIndexId);
-                            book.setLastIndexName(lastIndexName);
-                            book.setLastIndexUpdateTime(currentDate);
-                            book.setCreateTime(currentDate);
-                            book.setUpdateTime(currentDate);
-
-                        }else{
-                            //老书更新
                         }
+                        bookIndex.setUpdateTime(currentDate);
 
-
+                        //计算总字数
-
+                        totalWordCount += wordCount;
-                        if(hasIndex != null){
-                            bookIndex.setId(hasIndex.getId());
-                            bookContent.setIndexId(hasIndex.getId());
-                        }
 
 
                     }
@@ -279,6 +264,20 @@ public class CrawlParser {
                 isFindIndex = indexIdMatch.find() & indexNameMatch.find();
             }
 
+            //判断是新书入库还是老书更新
+            if (hasIndexs.size() == 0) {
+                //新书入库
+
+                //设置小说基础信息
+                book.setWordCount(totalWordCount);
+                book.setLastIndexId(lastIndexId);
+                book.setLastIndexName(lastIndexName);
+                book.setLastIndexUpdateTime(currentDate);
+                book.setCreateTime(currentDate);
+
+            }
+            book.setUpdateTime(currentDate);
+
             if (indexList.size() == contentList.size() && indexList.size() > 0) {
 
                 result.put(BOOK_INDEX_LIST_KEY, indexList);
@@ -288,6 +287,7 @@ public class CrawlParser {
 
         }
 
+
         return result;
     }
 

novel-crawl/src/main/java/com/java2nb/novel/service/BookService.java

@@ -65,8 +65,7 @@ public interface BookService {
      * @param book 小说数据
      * @param bookIndexList 目录集合
      * @param bookContentList 内容集合
-     * @param existBookIndexMap  已存在的章节Map
+     * @param existBookIndexMap  已存在的章节Map   */
-     * */
     void updateBookAndIndexAndContent(Book book,  List<BookIndex> bookIndexList, List<BookContent> bookContentList, Map<Integer, BookIndex> existBookIndexMap);
 
     /**

novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java

@@ -1,6 +1,5 @@
 package com.java2nb.novel.service.impl;
 
-import com.java2nb.novel.core.utils.IdWorker;
 import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.entity.BookContent;
 import com.java2nb.novel.entity.BookIndex;
@@ -79,10 +78,6 @@ public class BookServiceImpl implements BookService {
 
             if(bookIndexList.size()>0) {
 
-                if (book.getId() == null) {
-                    book.setId(new IdWorker().nextId());
-                }
-
                 //保存小说主表
 
                 bookMapper.insertSelective(book);
@@ -128,30 +123,14 @@ public class BookServiceImpl implements BookService {
             BookIndex bookIndex = bookIndexList.get(i);
             BookContent bookContent = bookContentList.get(i);
 
-            //插入或更新目录
-            Integer wordCount = bookContent.getContent().length();
-            bookIndex.setWordCount(wordCount);
-            bookIndex.setUpdateTime(currentDate);
 
-            if(bookIndex.getId() == null) {
+            if(!existBookIndexMap.containsKey(bookIndex.getIndexNum())) {
                 //插入
-                bookIndex.setBookId(book.getId());
-                Long indexId = new IdWorker().nextId();
-                bookIndex.setId(indexId);
-                bookIndex.setCreateTime(currentDate);
                 bookIndexMapper.insertSelective(bookIndex);
-            }else{
-                //更新
-                bookIndexMapper.updateByPrimaryKeySelective(bookIndex);
-            }
-
-            if(bookContent.getIndexId() == null) {
-                //插入
-                bookContent.setIndexId(bookIndex.getId());
                 bookContentMapper.insertSelective(bookContent);
             }else{
                 //更新
-
+                bookIndexMapper.updateByPrimaryKeySelective(bookIndex);
                 bookContentMapper.update(update(BookContentDynamicSqlSupport.bookContent)
                         .set(BookContentDynamicSqlSupport.content)
                         .equalTo(bookContent.getContent())
@@ -160,6 +139,7 @@ public class BookServiceImpl implements BookService {
                         .render(RenderingStrategies.MYBATIS3));
             }
 
+
         }
 
         //更新小说主表
@@ -174,7 +154,6 @@ public class BookServiceImpl implements BookService {
                 book.setLastIndexUpdateTime(currentDate);
             }
         }
-        book.setUpdateTime(currentDate);
         book.setBookName(null);
         book.setAuthorName(null);
         if(Constants.VISIT_COUNT_DEFAULT.equals(book.getVisitCount())) {

novel-crawl/src/main/resources/mybatis/mapping/BookMapper.xml

@@ -15,8 +15,7 @@
 
     <select id="queryTotalWordCount" parameterType="long" resultType="int">
 
-        select sum(t2.word_count) from book t1 inner join book_index t2
+        select sum(word_count) from book_index where book_id = #{bookId}
-        on t1.id = t2.book_id and t1.id = #{bookId}
     </select>
 
     <update id="updateCrawlLastTime">