更新策略优化

novel-front/pom.xml

@@ -109,6 +109,41 @@
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
             </plugin>
+            <!--<plugin>
+                <groupId>org.mybatis.generator</groupId>
+                <artifactId>mybatis-generator-maven-plugin</artifactId>
+                <version>1.3.2</version>
+                <configuration>
+                    <configurationFile>src/main/resources/mybatis/generatorConfig.xml</configurationFile>
+                    <verbose>false</verbose>
+                    <overwrite>false</overwrite>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>Generate MyBatis Artifacts</id>
+                        <goals>
+                            <goal>generate</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <dependencies>
+                    <dependency>
+                        <groupId>mysql</groupId>
+                        <artifactId>mysql-connector-java</artifactId>
+                        <version>${mysql.version}</version>
+                    </dependency>
+                    &lt;!&ndash;<dependency>
+                        <groupId>org.mybatis.generator</groupId>
+                        <artifactId>mybatis-generator-core</artifactId>
+                        <version>1.3.5</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.mybatis</groupId>
+                        <artifactId>mybatis</artifactId>
+                        <version>3.4.2</version>
+                    </dependency>&ndash;&gt;
+                </dependencies>
+            </plugin>-->
         </plugins>
     </build>
     <repositories>

novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java

@@ -5,20 +5,14 @@ import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import xyz.zinglizingli.books.core.utils.Constants;
 import xyz.zinglizingli.books.mapper.BookParseLogMapper;
-import xyz.zinglizingli.books.po.Book;
+import xyz.zinglizingli.books.po.*;
-import xyz.zinglizingli.books.po.BookContent;
-import xyz.zinglizingli.books.po.BookIndex;
-import xyz.zinglizingli.books.po.BookParseLog;
 import xyz.zinglizingli.books.service.BookService;
 import xyz.zinglizingli.books.core.utils.CatUtil;
 import xyz.zinglizingli.common.utils.ExcutorUtils;
 import xyz.zinglizingli.common.utils.RestTemplateUtil;
 
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -37,69 +31,94 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
     @Override
     public void parse() {
 
-        for(int page = 1; page<= Constants.UPDATE_PAGES_ONCE; page++) {
+        Map<Integer,Date> cat2Date = bookService.queryLastUpdateTime();
-            String catBookListUrl = getListPageUrl().replace("{0}", "0").replace("{1}", page+"");
+        Map<Integer,Date> newCat2Date = new HashMap<>();
-            String forObject = RestTemplateUtil.getBodyByUtf8(catBookListUrl);
+        for(int i=1;i<=7;i++) {
-            if (forObject != null) {
+            Date lastUpdateTime = cat2Date.get(i);
-                //解析第一页书籍的数据
+            Date updateTime = lastUpdateTime;
-                Pattern bookPatten = compile(getBookUrlPattern());
+            int page = 1;
+            do{
+                String catBookListUrl = getListPageUrl().replace("{0}", "0").replace("{1}", page + "");
+                page++;
+                String forObject = RestTemplateUtil.getBodyByUtf8(catBookListUrl);
+                if (forObject != null) {
+                    //解析第一页书籍的数据
+                    Pattern bookPatten = compile(getBookUrlPattern());
 
-                Matcher bookMatcher = bookPatten.matcher(forObject);
+                    Matcher bookMatcher = bookPatten.matcher(forObject);
 
-                boolean isFind = bookMatcher.find();
+                    boolean isFind = bookMatcher.find();
-                Pattern scorePatten = compile(getScorePattern());
+                    Pattern scorePatten = compile(getScorePattern());
-                Matcher scoreMatch = scorePatten.matcher(forObject);
+                    Matcher scoreMatch = scorePatten.matcher(forObject);
-                boolean scoreFind = scoreMatch.find();
+                    boolean scoreFind = scoreMatch.find();
 
-                Pattern bookNamePatten = compile(getBookNamePattern());
+                    Pattern bookNamePatten = compile(getBookNamePattern());
 
-                Matcher bookNameMatch = bookNamePatten.matcher(forObject);
+                    Matcher bookNameMatch = bookNamePatten.matcher(forObject);
 
-                Pattern authorPatten = compile(getAuthorPattern());
+                    Pattern authorPatten = compile(getAuthorPattern());
 
-                Matcher authorMatch = authorPatten.matcher(forObject);
+                    Matcher authorMatch = authorPatten.matcher(forObject);
 
-                boolean isBookNameMatch = bookNameMatch.find();
+                    boolean isBookNameMatch = bookNameMatch.find();
 
-                while (isFind && scoreFind && isBookNameMatch && authorMatch.find()) {
+                    while (isFind && scoreFind && isBookNameMatch && authorMatch.find() && updateTime.getTime()>=lastUpdateTime.getTime()) {
 
-                    try {
+                        try {
-                        Float score = Float.parseFloat(scoreMatch.group(1));
+                            Float score = Float.parseFloat(scoreMatch.group(1));
 
-                        if (score < getLowestScore()) {
+                            if (score < getLowestScore()) {
-                            continue;
+                                continue;
-                        }
+                            }
 
-                        String bokNum = bookMatcher.group(1);
+                            String bokNum = bookMatcher.group(1);
-                        String bookUrl = getIndexUrl() + "/" + bokNum + "/";
+                            String bookUrl = getIndexUrl() + "/" + bokNum + "/";
 
-                        String bookName = bookNameMatch.group(1);
+                            String bookName = bookNameMatch.group(1);
 
-                        String author = authorMatch.group(1);
+                            String author = authorMatch.group(1);
 
-                        Boolean hasBook = bookService.hasBook(bookName, author);
+                            Boolean hasBook = bookService.hasBook(bookName, author);
 
-                        if(hasBook) {
+                            if (hasBook) {
 
-                            bookService.addBookParseLog(bookUrl, bookName, score);
+                                bookService.addBookParseLog(bookUrl, bookName, score);
+                            }
+
+                            String body = RestTemplateUtil.getBodyByUtf8(bookUrl);
+                            if (body != null) {
+                                Pattern updateTimePatten = compile(getUpdateTimePattern());
+                                Matcher updateTimeMatch = updateTimePatten.matcher(body);
+                                if (updateTimeMatch.find()) {
+                                    String updateTimeStr = updateTimeMatch.group(1);
+                                    SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
+                                    updateTime = format.parse(updateTimeStr);
+                                    if(!newCat2Date.containsKey(i)) {
+                                        newCat2Date.put(i, updateTime);
+                                    }
+
+
+                                }
+                            }
+
+
+                        } catch (Exception e) {
+
+                            log.error(e.getMessage(), e);
+
+                        } finally {
+                            bookMatcher.find();
+                            isFind = bookMatcher.find();
+                            scoreFind = scoreMatch.find();
+                            isBookNameMatch = bookNameMatch.find();
                         }
 
 
-                    } catch (Exception e) {
-
-                        log.error(e.getMessage(), e);
-
-                    } finally {
-                        bookMatcher.find();
-                        isFind = bookMatcher.find();
-                        scoreFind = scoreMatch.find();
-                        isBookNameMatch = bookNameMatch.find();
                     }
-
-
                 }
-            }
+            }while (updateTime.getTime()>=lastUpdateTime.getTime());
         }
+        bookService.updateBookUpdateTimeLog(newCat2Date);
 
     }
 

novel-front/src/main/java/xyz/zinglizingli/books/core/listener/StartListener.java

@@ -66,7 +66,7 @@ public class StartListener implements ServletContextListener {
 
                         log.info("updateBooks执行中。。。。。。。。。。。。");
                         crawlSource.update();
-                        Thread.sleep(1000 * 60 * 10);
+                        Thread.sleep(new Float(1000 * 60 * bookUpdatePeriod).longValue());
                     } catch (Exception e) {
                         log.error(e.getMessage(), e);
                     }

novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookUpdateTimeLogMapper.java

@@ -0,0 +1,30 @@
+package xyz.zinglizingli.books.mapper;
+
+import java.util.List;
+import org.apache.ibatis.annotations.Param;
+import xyz.zinglizingli.books.po.BookUpdateTimeLog;
+import xyz.zinglizingli.books.po.BookUpdateTimeLogExample;
+
+public interface BookUpdateTimeLogMapper {
+    int countByExample(BookUpdateTimeLogExample example);
+
+    int deleteByExample(BookUpdateTimeLogExample example);
+
+    int deleteByPrimaryKey(Integer id);
+
+    int insert(BookUpdateTimeLog record);
+
+    int insertSelective(BookUpdateTimeLog record);
+
+    List<BookUpdateTimeLog> selectByExample(BookUpdateTimeLogExample example);
+
+    BookUpdateTimeLog selectByPrimaryKey(Integer id);
+
+    int updateByExampleSelective(@Param("record") BookUpdateTimeLog record, @Param("example") BookUpdateTimeLogExample example);
+
+    int updateByExample(@Param("record") BookUpdateTimeLog record, @Param("example") BookUpdateTimeLogExample example);
+
+    int updateByPrimaryKeySelective(BookUpdateTimeLog record);
+
+    int updateByPrimaryKey(BookUpdateTimeLog record);
+}

novel-front/src/main/java/xyz/zinglizingli/books/po/BookUpdateTimeLog.java

@@ -0,0 +1,35 @@
+package xyz.zinglizingli.books.po;
+
+import java.util.Date;
+
+public class BookUpdateTimeLog {
+    private Integer id;
+
+    private Integer bookCatId;
+
+    private Date lastUpdateTime;
+
+    public Integer getId() {
+        return id;
+    }
+
+    public void setId(Integer id) {
+        this.id = id;
+    }
+
+    public Integer getBookCatId() {
+        return bookCatId;
+    }
+
+    public void setBookCatId(Integer bookCatId) {
+        this.bookCatId = bookCatId;
+    }
+
+    public Date getLastUpdateTime() {
+        return lastUpdateTime;
+    }
+
+    public void setLastUpdateTime(Date lastUpdateTime) {
+        this.lastUpdateTime = lastUpdateTime;
+    }
+}

novel-front/src/main/java/xyz/zinglizingli/books/po/BookUpdateTimeLogExample.java

@@ -0,0 +1,381 @@
+package xyz.zinglizingli.books.po;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class BookUpdateTimeLogExample {
+    protected String orderByClause;
+
+    protected boolean distinct;
+
+    protected List<Criteria> oredCriteria;
+
+    public BookUpdateTimeLogExample() {
+        oredCriteria = new ArrayList<Criteria>();
+    }
+
+    public void setOrderByClause(String orderByClause) {
+        this.orderByClause = orderByClause;
+    }
+
+    public String getOrderByClause() {
+        return orderByClause;
+    }
+
+    public void setDistinct(boolean distinct) {
+        this.distinct = distinct;
+    }
+
+    public boolean isDistinct() {
+        return distinct;
+    }
+
+    public List<Criteria> getOredCriteria() {
+        return oredCriteria;
+    }
+
+    public void or(Criteria criteria) {
+        oredCriteria.add(criteria);
+    }
+
+    public Criteria or() {
+        Criteria criteria = createCriteriaInternal();
+        oredCriteria.add(criteria);
+        return criteria;
+    }
+
+    public Criteria createCriteria() {
+        Criteria criteria = createCriteriaInternal();
+        if (oredCriteria.size() == 0) {
+            oredCriteria.add(criteria);
+        }
+        return criteria;
+    }
+
+    protected Criteria createCriteriaInternal() {
+        Criteria criteria = new Criteria();
+        return criteria;
+    }
+
+    public void clear() {
+        oredCriteria.clear();
+        orderByClause = null;
+        distinct = false;
+    }
+
+    protected abstract static class GeneratedCriteria {
+        protected List<Criterion> criteria;
+
+        protected GeneratedCriteria() {
+            super();
+            criteria = new ArrayList<Criterion>();
+        }
+
+        public boolean isValid() {
+            return criteria.size() > 0;
+        }
+
+        public List<Criterion> getAllCriteria() {
+            return criteria;
+        }
+
+        public List<Criterion> getCriteria() {
+            return criteria;
+        }
+
+        protected void addCriterion(String condition) {
+            if (condition == null) {
+                throw new RuntimeException("Value for condition cannot be null");
+            }
+            criteria.add(new Criterion(condition));
+        }
+
+        protected void addCriterion(String condition, Object value, String property) {
+            if (value == null) {
+                throw new RuntimeException("Value for " + property + " cannot be null");
+            }
+            criteria.add(new Criterion(condition, value));
+        }
+
+        protected void addCriterion(String condition, Object value1, Object value2, String property) {
+            if (value1 == null || value2 == null) {
+                throw new RuntimeException("Between values for " + property + " cannot be null");
+            }
+            criteria.add(new Criterion(condition, value1, value2));
+        }
+
+        public Criteria andIdIsNull() {
+            addCriterion("id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdIsNotNull() {
+            addCriterion("id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdEqualTo(Integer value) {
+            addCriterion("id =", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotEqualTo(Integer value) {
+            addCriterion("id <>", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdGreaterThan(Integer value) {
+            addCriterion("id >", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdGreaterThanOrEqualTo(Integer value) {
+            addCriterion("id >=", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdLessThan(Integer value) {
+            addCriterion("id <", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdLessThanOrEqualTo(Integer value) {
+            addCriterion("id <=", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdIn(List<Integer> values) {
+            addCriterion("id in", values, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotIn(List<Integer> values) {
+            addCriterion("id not in", values, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdBetween(Integer value1, Integer value2) {
+            addCriterion("id between", value1, value2, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotBetween(Integer value1, Integer value2) {
+            addCriterion("id not between", value1, value2, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdIsNull() {
+            addCriterion("book_cat_id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdIsNotNull() {
+            addCriterion("book_cat_id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdEqualTo(Integer value) {
+            addCriterion("book_cat_id =", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdNotEqualTo(Integer value) {
+            addCriterion("book_cat_id <>", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdGreaterThan(Integer value) {
+            addCriterion("book_cat_id >", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdGreaterThanOrEqualTo(Integer value) {
+            addCriterion("book_cat_id >=", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdLessThan(Integer value) {
+            addCriterion("book_cat_id <", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdLessThanOrEqualTo(Integer value) {
+            addCriterion("book_cat_id <=", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdIn(List<Integer> values) {
+            addCriterion("book_cat_id in", values, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdNotIn(List<Integer> values) {
+            addCriterion("book_cat_id not in", values, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdBetween(Integer value1, Integer value2) {
+            addCriterion("book_cat_id between", value1, value2, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdNotBetween(Integer value1, Integer value2) {
+            addCriterion("book_cat_id not between", value1, value2, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeIsNull() {
+            addCriterion("last_update_time is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeIsNotNull() {
+            addCriterion("last_update_time is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeEqualTo(Date value) {
+            addCriterion("last_update_time =", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeNotEqualTo(Date value) {
+            addCriterion("last_update_time <>", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeGreaterThan(Date value) {
+            addCriterion("last_update_time >", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeGreaterThanOrEqualTo(Date value) {
+            addCriterion("last_update_time >=", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeLessThan(Date value) {
+            addCriterion("last_update_time <", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeLessThanOrEqualTo(Date value) {
+            addCriterion("last_update_time <=", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeIn(List<Date> values) {
+            addCriterion("last_update_time in", values, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeNotIn(List<Date> values) {
+            addCriterion("last_update_time not in", values, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeBetween(Date value1, Date value2) {
+            addCriterion("last_update_time between", value1, value2, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeNotBetween(Date value1, Date value2) {
+            addCriterion("last_update_time not between", value1, value2, "lastUpdateTime");
+            return (Criteria) this;
+        }
+    }
+
+    public static class Criteria extends GeneratedCriteria {
+
+        protected Criteria() {
+            super();
+        }
+    }
+
+    public static class Criterion {
+        private String condition;
+
+        private Object value;
+
+        private Object secondValue;
+
+        private boolean noValue;
+
+        private boolean singleValue;
+
+        private boolean betweenValue;
+
+        private boolean listValue;
+
+        private String typeHandler;
+
+        public String getCondition() {
+            return condition;
+        }
+
+        public Object getValue() {
+            return value;
+        }
+
+        public Object getSecondValue() {
+            return secondValue;
+        }
+
+        public boolean isNoValue() {
+            return noValue;
+        }
+
+        public boolean isSingleValue() {
+            return singleValue;
+        }
+
+        public boolean isBetweenValue() {
+            return betweenValue;
+        }
+
+        public boolean isListValue() {
+            return listValue;
+        }
+
+        public String getTypeHandler() {
+            return typeHandler;
+        }
+
+        protected Criterion(String condition) {
+            super();
+            this.condition = condition;
+            this.typeHandler = null;
+            this.noValue = true;
+        }
+
+        protected Criterion(String condition, Object value, String typeHandler) {
+            super();
+            this.condition = condition;
+            this.value = value;
+            this.typeHandler = typeHandler;
+            if (value instanceof List<?>) {
+                this.listValue = true;
+            } else {
+                this.singleValue = true;
+            }
+        }
+
+        protected Criterion(String condition, Object value) {
+            this(condition, value, null);
+        }
+
+        protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
+            super();
+            this.condition = condition;
+            this.value = value;
+            this.secondValue = secondValue;
+            this.typeHandler = typeHandler;
+            this.betweenValue = true;
+        }
+
+        protected Criterion(String condition, Object value, Object secondValue) {
+            this(condition, value, secondValue, null);
+        }
+    }
+}

novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java

@@ -3,30 +3,20 @@ package xyz.zinglizingli.books.service;
 import com.github.pagehelper.PageHelper;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.codec.Charsets;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.DateUtils;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.Resource;
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import tk.mybatis.orderbyhelper.OrderByHelper;
 import xyz.zinglizingli.books.core.constant.CacheKeyConstans;
 import xyz.zinglizingli.books.core.enums.PicSaveType;
+import xyz.zinglizingli.books.core.utils.Constants;
 import xyz.zinglizingli.books.mapper.*;
 import xyz.zinglizingli.books.po.*;
-import xyz.zinglizingli.books.core.utils.Constants;
+import xyz.zinglizingli.common.cache.CommonCacheUtil;
 import xyz.zinglizingli.common.utils.FileUtil;
 import xyz.zinglizingli.common.utils.SpringUtil;
-import xyz.zinglizingli.common.utils.UUIDUtils;
-import xyz.zinglizingli.common.cache.CommonCacheUtil;
-import xyz.zinglizingli.common.utils.RestTemplateUtil;
 
-import java.io.*;
 import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -51,6 +41,8 @@ public class BookService {
 
     private final BookParseLogMapper bookParseLogMapper;
 
+    private final BookUpdateTimeLogMapper bookUpdateTimeLogMapper;
+
     private final CommonCacheUtil cacheUtil;
 
 
@@ -491,4 +483,31 @@ public class BookService {
         example.createCriteria().andBookNameEqualTo(bookName).andAuthorEqualTo(author);
         return bookMapper.countByExample(example)>0;
     }
+
+    /**
+     * 查询分类更新时间映射信息
+     * */
+    public Map<Integer, Date> queryLastUpdateTime() {
+        List<BookUpdateTimeLog> list = bookUpdateTimeLogMapper.selectByExample(new BookUpdateTimeLogExample());
+
+        return list.stream().collect(Collectors.toMap(BookUpdateTimeLog::getBookCatId, BookUpdateTimeLog::getLastUpdateTime,(key1, key2) -> key2));
+
+    }
+
+    /**
+     * 更新分类时间日志
+     * */
+    public void updateBookUpdateTimeLog(Map<Integer, Date> cat2Date) {
+        if(cat2Date.size()>0) {
+            Set<Map.Entry<Integer, Date>> entries = cat2Date.entrySet();
+            for(Map.Entry<Integer, Date> entry : entries){
+                BookUpdateTimeLogExample example = new BookUpdateTimeLogExample();
+                example.createCriteria().andBookCatIdEqualTo(entry.getKey());
+                BookUpdateTimeLog entity = new BookUpdateTimeLog();
+                entity.setLastUpdateTime(entry.getValue());
+                bookUpdateTimeLogMapper.updateByExampleSelective(entity,example);
+
+            }
+        }
+    }
 }

novel-front/src/main/resources/application.yml

@@ -69,7 +69,7 @@ books:
   #爬取小说数据的最低评分
   lowestScore: 6.0
   #小说的更新间隔（分）
-  updatePeriod: 5
+  updatePeriod: 1
 
 #爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说  更多网站解析中，敬请期待
 crawl:

novel-front/src/main/resources/mybatis/generatorConfig.xml

@@ -42,7 +42,7 @@
 			<property name="enableSubPackages" value="false" />
 		</javaClientGenerator>
 
-		<table  tableName="book_parse_log"/>
+		<table  tableName="book_update_time_log"/>
 		
 		<!-- 指定数据库表 -->
 		<!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->

novel-front/src/main/resources/mybatis/mapping/BookUpdateTimeLogMapper.xml

@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="xyz.zinglizingli.books.mapper.BookUpdateTimeLogMapper" >
+  <resultMap id="BaseResultMap" type="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    <id column="id" property="id" jdbcType="INTEGER" />
+    <result column="book_cat_id" property="bookCatId" jdbcType="INTEGER" />
+    <result column="last_update_time" property="lastUpdateTime" jdbcType="TIMESTAMP" />
+  </resultMap>
+  <sql id="Example_Where_Clause" >
+    <where >
+      <foreach collection="oredCriteria" item="criteria" separator="or" >
+        <if test="criteria.valid" >
+          <trim prefix="(" suffix=")" prefixOverrides="and" >
+            <foreach collection="criteria.criteria" item="criterion" >
+              <choose >
+                <when test="criterion.noValue" >
+                  and ${criterion.condition}
+                </when>
+                <when test="criterion.singleValue" >
+                  and ${criterion.condition} #{criterion.value}
+                </when>
+                <when test="criterion.betweenValue" >
+                  and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
+                </when>
+                <when test="criterion.listValue" >
+                  and ${criterion.condition}
+                  <foreach collection="criterion.value" item="listItem" open="(" close=")" separator="," >
+                    #{listItem}
+                  </foreach>
+                </when>
+              </choose>
+            </foreach>
+          </trim>
+        </if>
+      </foreach>
+    </where>
+  </sql>
+  <sql id="Update_By_Example_Where_Clause" >
+    <where >
+      <foreach collection="example.oredCriteria" item="criteria" separator="or" >
+        <if test="criteria.valid" >
+          <trim prefix="(" suffix=")" prefixOverrides="and" >
+            <foreach collection="criteria.criteria" item="criterion" >
+              <choose >
+                <when test="criterion.noValue" >
+                  and ${criterion.condition}
+                </when>
+                <when test="criterion.singleValue" >
+                  and ${criterion.condition} #{criterion.value}
+                </when>
+                <when test="criterion.betweenValue" >
+                  and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
+                </when>
+                <when test="criterion.listValue" >
+                  and ${criterion.condition}
+                  <foreach collection="criterion.value" item="listItem" open="(" close=")" separator="," >
+                    #{listItem}
+                  </foreach>
+                </when>
+              </choose>
+            </foreach>
+          </trim>
+        </if>
+      </foreach>
+    </where>
+  </sql>
+  <sql id="Base_Column_List" >
+    id, book_cat_id, last_update_time
+  </sql>
+  <select id="selectByExample" resultMap="BaseResultMap" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLogExample" >
+    select
+    <if test="distinct" >
+      distinct
+    </if>
+    <include refid="Base_Column_List" />
+    from book_update_time_log
+    <if test="_parameter != null" >
+      <include refid="Example_Where_Clause" />
+    </if>
+    <if test="orderByClause != null" >
+      order by ${orderByClause}
+    </if>
+  </select>
+  <select id="selectByPrimaryKey" resultMap="BaseResultMap" parameterType="java.lang.Integer" >
+    select 
+    <include refid="Base_Column_List" />
+    from book_update_time_log
+    where id = #{id,jdbcType=INTEGER}
+  </select>
+  <delete id="deleteByPrimaryKey" parameterType="java.lang.Integer" >
+    delete from book_update_time_log
+    where id = #{id,jdbcType=INTEGER}
+  </delete>
+  <delete id="deleteByExample" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLogExample" >
+    delete from book_update_time_log
+    <if test="_parameter != null" >
+      <include refid="Example_Where_Clause" />
+    </if>
+  </delete>
+  <insert id="insert" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    insert into book_update_time_log (id, book_cat_id, last_update_time
+      )
+    values (#{id,jdbcType=INTEGER}, #{bookCatId,jdbcType=INTEGER}, #{lastUpdateTime,jdbcType=TIMESTAMP}
+      )
+  </insert>
+  <insert id="insertSelective" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    insert into book_update_time_log
+    <trim prefix="(" suffix=")" suffixOverrides="," >
+      <if test="id != null" >
+        id,
+      </if>
+      <if test="bookCatId != null" >
+        book_cat_id,
+      </if>
+      <if test="lastUpdateTime != null" >
+        last_update_time,
+      </if>
+    </trim>
+    <trim prefix="values (" suffix=")" suffixOverrides="," >
+      <if test="id != null" >
+        #{id,jdbcType=INTEGER},
+      </if>
+      <if test="bookCatId != null" >
+        #{bookCatId,jdbcType=INTEGER},
+      </if>
+      <if test="lastUpdateTime != null" >
+        #{lastUpdateTime,jdbcType=TIMESTAMP},
+      </if>
+    </trim>
+  </insert>
+  <select id="countByExample" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLogExample" resultType="java.lang.Integer" >
+    select count(*) from book_update_time_log
+    <if test="_parameter != null" >
+      <include refid="Example_Where_Clause" />
+    </if>
+  </select>
+  <update id="updateByExampleSelective" parameterType="map" >
+    update book_update_time_log
+    <set >
+      <if test="record.id != null" >
+        id = #{record.id,jdbcType=INTEGER},
+      </if>
+      <if test="record.bookCatId != null" >
+        book_cat_id = #{record.bookCatId,jdbcType=INTEGER},
+      </if>
+      <if test="record.lastUpdateTime != null" >
+        last_update_time = #{record.lastUpdateTime,jdbcType=TIMESTAMP},
+      </if>
+    </set>
+    <if test="_parameter != null" >
+      <include refid="Update_By_Example_Where_Clause" />
+    </if>
+  </update>
+  <update id="updateByExample" parameterType="map" >
+    update book_update_time_log
+    set id = #{record.id,jdbcType=INTEGER},
+      book_cat_id = #{record.bookCatId,jdbcType=INTEGER},
+      last_update_time = #{record.lastUpdateTime,jdbcType=TIMESTAMP}
+    <if test="_parameter != null" >
+      <include refid="Update_By_Example_Where_Clause" />
+    </if>
+  </update>
+  <update id="updateByPrimaryKeySelective" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    update book_update_time_log
+    <set >
+      <if test="bookCatId != null" >
+        book_cat_id = #{bookCatId,jdbcType=INTEGER},
+      </if>
+      <if test="lastUpdateTime != null" >
+        last_update_time = #{lastUpdateTime,jdbcType=TIMESTAMP},
+      </if>
+    </set>
+    where id = #{id,jdbcType=INTEGER}
+  </update>
+  <update id="updateByPrimaryKey" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    update book_update_time_log
+    set book_cat_id = #{bookCatId,jdbcType=INTEGER},
+      last_update_time = #{lastUpdateTime,jdbcType=TIMESTAMP}
+    where id = #{id,jdbcType=INTEGER}
+  </update>
+</mapper>

sql/2020-01-17.sql

@@ -0,0 +1,17 @@
+
+DROP TABLE IF EXISTS `book_update_time_log`;
+CREATE TABLE `book_update_time_log` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `book_cat_id` int(11) NOT NULL,
+  `last_update_time` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_key_catid` (`book_cat_id`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4;
+
+INSERT INTO `book_update_time_log` VALUES ('1', '1', NOW());
+INSERT INTO `book_update_time_log` VALUES ('2', '2', NOW());
+INSERT INTO `book_update_time_log` VALUES ('3', '3', NOW());
+INSERT INTO `book_update_time_log` VALUES ('4', '4', NOW());
+INSERT INTO `book_update_time_log` VALUES ('5', '5', NOW());
+INSERT INTO `book_update_time_log` VALUES ('6', '6', NOW());
+INSERT INTO `book_update_time_log` VALUES ('7', '7', NOW());

sql/books.sql

@@ -1024,3 +1024,19 @@ CREATE TABLE `book_parse_log` (
   UNIQUE KEY `uq_key_bookurl` (`book_url`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
 
+DROP TABLE IF EXISTS `book_update_time_log`;
+CREATE TABLE `book_update_time_log` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `book_cat_id` int(11) NOT NULL,
+  `last_update_time` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_key_catid` (`book_cat_id`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4;
+
+INSERT INTO `book_update_time_log` VALUES ('1', '1', NOW());
+INSERT INTO `book_update_time_log` VALUES ('2', '2', NOW());
+INSERT INTO `book_update_time_log` VALUES ('3', '3', NOW());
+INSERT INTO `book_update_time_log` VALUES ('4', '4', NOW());
+INSERT INTO `book_update_time_log` VALUES ('5', '5', NOW());
+INSERT INTO `book_update_time_log` VALUES ('6', '6', NOW());
+INSERT INTO `book_update_time_log` VALUES ('7', '7', NOW());