update

novel-front/pom.xml

@@ -10,7 +10,7 @@
     </parent>
     <groupId>xyz.zinglizingli</groupId>
     <artifactId>novel-front</artifactId>
-    <version>2.2.0.beta</version>
+    <version>2.3.0.beta</version>
     <name>novel-front</name>
     <description>小说精品楼-前台web网站</description>
 
@@ -109,6 +109,41 @@
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
             </plugin>
+            <!--<plugin>
+                <groupId>org.mybatis.generator</groupId>
+                <artifactId>mybatis-generator-maven-plugin</artifactId>
+                <version>1.3.2</version>
+                <configuration>
+                    <configurationFile>src/main/resources/mybatis/generatorConfig.xml</configurationFile>
+                    <verbose>false</verbose>
+                    <overwrite>false</overwrite>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>Generate MyBatis Artifacts</id>
+                        <goals>
+                            <goal>generate</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <dependencies>
+                    <dependency>
+                        <groupId>mysql</groupId>
+                        <artifactId>mysql-connector-java</artifactId>
+                        <version>${mysql.version}</version>
+                    </dependency>
+                    &lt;!&ndash;<dependency>
+                        <groupId>org.mybatis.generator</groupId>
+                        <artifactId>mybatis-generator-core</artifactId>
+                        <version>1.3.5</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.mybatis</groupId>
+                        <artifactId>mybatis</artifactId>
+                        <version>3.4.2</version>
+                    </dependency>&ndash;&gt;
+                </dependencies>
+            </plugin>-->
         </plugins>
     </build>
     <repositories>

novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlBaishuzhaiConfig.java

@@ -0,0 +1,29 @@
+package xyz.zinglizingli.books.core.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import xyz.zinglizingli.books.core.crawl.BaseHtmlCrawlSource;
+import xyz.zinglizingli.books.core.crawl.BiquCrawlSource;
+
+/**
+ * @author 11797
+ */
+@Slf4j
+@Configuration
+public class CrawlBaishuzhaiConfig {
+
+
+    @Bean
+    @Primary //必须加此注解，不然报错，下一个类则不需要添加
+    @ConfigurationProperties(prefix = "baishuzhai.crawlsource") // prefix值必须是application.yml中对应属性的前缀
+    @ConditionalOnProperty(prefix = "crawl.website",name = "type",havingValue = "4")
+    public BaseHtmlCrawlSource dingdianCrawlSource() {
+        return new BiquCrawlSource();
+    }
+
+
+}

novel-front/src/main/java/xyz/zinglizingli/books/core/config/CrawlDingdianConfig.java

@@ -18,7 +18,6 @@ public class CrawlDingdianConfig {
 
 
     @Bean
-    @Primary //必须加此注解，不然报错，下一个类则不需要添加
     @ConfigurationProperties(prefix = "dingdian.crawlsource") // prefix值必须是application.yml中对应属性的前缀
     @ConditionalOnProperty(prefix = "crawl.website",name = "type",havingValue = "3")
     public BaseHtmlCrawlSource dingdianCrawlSource() {

novel-front/src/main/java/xyz/zinglizingli/books/core/crawl/BiquCrawlSource.java

@@ -5,20 +5,14 @@ import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import xyz.zinglizingli.books.core.utils.Constants;
 import xyz.zinglizingli.books.mapper.BookParseLogMapper;
-import xyz.zinglizingli.books.po.Book;
+import xyz.zinglizingli.books.po.*;
-import xyz.zinglizingli.books.po.BookContent;
-import xyz.zinglizingli.books.po.BookIndex;
-import xyz.zinglizingli.books.po.BookParseLog;
 import xyz.zinglizingli.books.service.BookService;
 import xyz.zinglizingli.books.core.utils.CatUtil;
 import xyz.zinglizingli.common.utils.ExcutorUtils;
 import xyz.zinglizingli.common.utils.RestTemplateUtil;
 
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -55,13 +49,9 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                 Matcher bookNameMatch = bookNamePatten.matcher(forObject);
 
-                Pattern authorPatten = compile(getAuthorPattern());
-
-                Matcher authorMatch = authorPatten.matcher(forObject);
-
                 boolean isBookNameMatch = bookNameMatch.find();
 
-                while (isFind && scoreFind && isBookNameMatch && authorMatch.find()) {
+                while (isFind && scoreFind && isBookNameMatch) {
 
                     try {
                         Float score = Float.parseFloat(scoreMatch.group(1));
@@ -75,14 +65,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                         String bookName = bookNameMatch.group(1);
 
-                        String author = authorMatch.group(1);
-
-                        Boolean hasBook = bookService.hasBook(bookName, author);
-
-                        if(hasBook) {
-
                         bookService.addBookParseLog(bookUrl, bookName, score);
-                        }
 
 
                     } catch (Exception e) {
@@ -106,7 +89,6 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
     @Override
     public void update() {
         List<BookParseLog> logs = bookService.queryBookParseLogs();
-        List<Long> successLogIds = new ArrayList<>();
         for (BookParseLog bookParseLog : logs) {
             try {
 
@@ -139,10 +121,16 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                 Pattern updateTimePatten = compile(getUpdateTimePattern());
                                 Matcher updateTimeMatch = updateTimePatten.matcher(body);
-                                if (updateTimeMatch.find()) {
+                                /*if (updateTimeMatch.find()) {
                                     String updateTimeStr = updateTimeMatch.group(1);
-                                    SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
+                                    SimpleDateFormat format ;
-                                    Date updateTime = format.parse(updateTimeStr);
+                                    if(updateTimeStr.length()>10){
+
+                                        format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
+                                    }else{
+                                        format = new SimpleDateFormat("yy-MM-dd");
+                                    }
+                                    Date updateTime = format.parse(updateTimeStr);*/
                                     Pattern picPatten = compile(getPicPattern());
                                     Matcher picMather = picPatten.matcher(body);
                                     if (picMather.find()) {
@@ -159,7 +147,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
                                         book.setScore(score > 10 ? 8.0f : score);
                                         book.setPicUrl(picSrc);
                                         book.setBookStatus(status);
-                                        book.setUpdateTime(updateTime);
+                                        book.setUpdateTime(new Date());
 
                                         List<BookIndex> indexList = new ArrayList<>();
                                         List<BookContent> contentList = new ArrayList<>();
@@ -222,7 +210,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                                     }
                                                 }
-                                                successLogIds.add(bookParseLog.getId());
+                                                bookService.deleteBookParseLog(bookParseLog.getId());
                                             }
 
 
@@ -231,7 +219,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
 
                                     }
 
-                                }
+                                //}
                             }
                         }
 
@@ -247,7 +235,7 @@ public class BiquCrawlSource extends BaseHtmlCrawlSource {
             }
         }
 
-        bookService.deleteBookParseLogs(successLogIds);
+
 
     }
 

novel-front/src/main/java/xyz/zinglizingli/books/core/listener/StartListener.java

@@ -31,6 +31,9 @@ public class StartListener implements ServletContextListener {
     @Value("${website.domain}")
     private String webSiteDomain;
 
+    @Value("${books.updatePeriod}")
+    private float bookUpdatePeriod;
+
     private final SeoConfig seoConfig;
 
 
@@ -47,10 +50,22 @@ public class StartListener implements ServletContextListener {
                 while (true) {
                     try {
 
-                        log.info("crawlBooks执行中。。。。。。。。。。。。");
+                        log.info("parseBooks执行中。。。。。。。。。。。。");
                         crawlSource.parse();
+                        Thread.sleep(new Float(1000 * 60 * bookUpdatePeriod).longValue());
+                    } catch (Exception e) {
+                        log.error(e.getMessage(), e);
+                    }
 
-                        Thread.sleep(1000 * 60 * 5);
+                }
+            }).start();
+
+            new Thread(() -> {
+                while (true) {
+                    try {
+
+                        log.info("updateBooks执行中。。。。。。。。。。。。");
+                        crawlSource.update();
                     } catch (Exception e) {
                         log.error(e.getMessage(), e);
                     }

novel-front/src/main/java/xyz/zinglizingli/books/core/schedule/UpdateBookSchedule.java

@@ -1,39 +0,0 @@
-package xyz.zinglizingli.books.core.schedule;
-
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Service;
-import xyz.zinglizingli.books.core.crawl.BaseCrawlSource;
-import xyz.zinglizingli.books.po.Book;
-import xyz.zinglizingli.books.service.BookService;
-
-import java.util.List;
-
-/**
- * 定时更新
- *
- * @author 11797
- */
-@Service
-@RequiredArgsConstructor
-@Slf4j
-public class UpdateBookSchedule {
-
-    private final BaseCrawlSource crawlSource;
-
-
-    /**
-     * 10分钟更新一次
-     */
-    @Scheduled(fixedRate = 1000 * 60 * 10)
-    public void updateBook() {
-
-        log.info("UpdateBookSchedule。。。。。。。。。。。。");
-
-        crawlSource.update();
-    }
-}

novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookUpdateTimeLogMapper.java

@@ -0,0 +1,30 @@
+package xyz.zinglizingli.books.mapper;
+
+import java.util.List;
+import org.apache.ibatis.annotations.Param;
+import xyz.zinglizingli.books.po.BookUpdateTimeLog;
+import xyz.zinglizingli.books.po.BookUpdateTimeLogExample;
+
+public interface BookUpdateTimeLogMapper {
+    int countByExample(BookUpdateTimeLogExample example);
+
+    int deleteByExample(BookUpdateTimeLogExample example);
+
+    int deleteByPrimaryKey(Integer id);
+
+    int insert(BookUpdateTimeLog record);
+
+    int insertSelective(BookUpdateTimeLog record);
+
+    List<BookUpdateTimeLog> selectByExample(BookUpdateTimeLogExample example);
+
+    BookUpdateTimeLog selectByPrimaryKey(Integer id);
+
+    int updateByExampleSelective(@Param("record") BookUpdateTimeLog record, @Param("example") BookUpdateTimeLogExample example);
+
+    int updateByExample(@Param("record") BookUpdateTimeLog record, @Param("example") BookUpdateTimeLogExample example);
+
+    int updateByPrimaryKeySelective(BookUpdateTimeLog record);
+
+    int updateByPrimaryKey(BookUpdateTimeLog record);
+}

novel-front/src/main/java/xyz/zinglizingli/books/po/BookUpdateTimeLog.java

@@ -0,0 +1,35 @@
+package xyz.zinglizingli.books.po;
+
+import java.util.Date;
+
+public class BookUpdateTimeLog {
+    private Integer id;
+
+    private Integer bookCatId;
+
+    private Date lastUpdateTime;
+
+    public Integer getId() {
+        return id;
+    }
+
+    public void setId(Integer id) {
+        this.id = id;
+    }
+
+    public Integer getBookCatId() {
+        return bookCatId;
+    }
+
+    public void setBookCatId(Integer bookCatId) {
+        this.bookCatId = bookCatId;
+    }
+
+    public Date getLastUpdateTime() {
+        return lastUpdateTime;
+    }
+
+    public void setLastUpdateTime(Date lastUpdateTime) {
+        this.lastUpdateTime = lastUpdateTime;
+    }
+}

novel-front/src/main/java/xyz/zinglizingli/books/po/BookUpdateTimeLogExample.java

@@ -0,0 +1,381 @@
+package xyz.zinglizingli.books.po;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class BookUpdateTimeLogExample {
+    protected String orderByClause;
+
+    protected boolean distinct;
+
+    protected List<Criteria> oredCriteria;
+
+    public BookUpdateTimeLogExample() {
+        oredCriteria = new ArrayList<Criteria>();
+    }
+
+    public void setOrderByClause(String orderByClause) {
+        this.orderByClause = orderByClause;
+    }
+
+    public String getOrderByClause() {
+        return orderByClause;
+    }
+
+    public void setDistinct(boolean distinct) {
+        this.distinct = distinct;
+    }
+
+    public boolean isDistinct() {
+        return distinct;
+    }
+
+    public List<Criteria> getOredCriteria() {
+        return oredCriteria;
+    }
+
+    public void or(Criteria criteria) {
+        oredCriteria.add(criteria);
+    }
+
+    public Criteria or() {
+        Criteria criteria = createCriteriaInternal();
+        oredCriteria.add(criteria);
+        return criteria;
+    }
+
+    public Criteria createCriteria() {
+        Criteria criteria = createCriteriaInternal();
+        if (oredCriteria.size() == 0) {
+            oredCriteria.add(criteria);
+        }
+        return criteria;
+    }
+
+    protected Criteria createCriteriaInternal() {
+        Criteria criteria = new Criteria();
+        return criteria;
+    }
+
+    public void clear() {
+        oredCriteria.clear();
+        orderByClause = null;
+        distinct = false;
+    }
+
+    protected abstract static class GeneratedCriteria {
+        protected List<Criterion> criteria;
+
+        protected GeneratedCriteria() {
+            super();
+            criteria = new ArrayList<Criterion>();
+        }
+
+        public boolean isValid() {
+            return criteria.size() > 0;
+        }
+
+        public List<Criterion> getAllCriteria() {
+            return criteria;
+        }
+
+        public List<Criterion> getCriteria() {
+            return criteria;
+        }
+
+        protected void addCriterion(String condition) {
+            if (condition == null) {
+                throw new RuntimeException("Value for condition cannot be null");
+            }
+            criteria.add(new Criterion(condition));
+        }
+
+        protected void addCriterion(String condition, Object value, String property) {
+            if (value == null) {
+                throw new RuntimeException("Value for " + property + " cannot be null");
+            }
+            criteria.add(new Criterion(condition, value));
+        }
+
+        protected void addCriterion(String condition, Object value1, Object value2, String property) {
+            if (value1 == null || value2 == null) {
+                throw new RuntimeException("Between values for " + property + " cannot be null");
+            }
+            criteria.add(new Criterion(condition, value1, value2));
+        }
+
+        public Criteria andIdIsNull() {
+            addCriterion("id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdIsNotNull() {
+            addCriterion("id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdEqualTo(Integer value) {
+            addCriterion("id =", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotEqualTo(Integer value) {
+            addCriterion("id <>", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdGreaterThan(Integer value) {
+            addCriterion("id >", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdGreaterThanOrEqualTo(Integer value) {
+            addCriterion("id >=", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdLessThan(Integer value) {
+            addCriterion("id <", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdLessThanOrEqualTo(Integer value) {
+            addCriterion("id <=", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdIn(List<Integer> values) {
+            addCriterion("id in", values, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotIn(List<Integer> values) {
+            addCriterion("id not in", values, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdBetween(Integer value1, Integer value2) {
+            addCriterion("id between", value1, value2, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotBetween(Integer value1, Integer value2) {
+            addCriterion("id not between", value1, value2, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdIsNull() {
+            addCriterion("book_cat_id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdIsNotNull() {
+            addCriterion("book_cat_id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdEqualTo(Integer value) {
+            addCriterion("book_cat_id =", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdNotEqualTo(Integer value) {
+            addCriterion("book_cat_id <>", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdGreaterThan(Integer value) {
+            addCriterion("book_cat_id >", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdGreaterThanOrEqualTo(Integer value) {
+            addCriterion("book_cat_id >=", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdLessThan(Integer value) {
+            addCriterion("book_cat_id <", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdLessThanOrEqualTo(Integer value) {
+            addCriterion("book_cat_id <=", value, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdIn(List<Integer> values) {
+            addCriterion("book_cat_id in", values, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdNotIn(List<Integer> values) {
+            addCriterion("book_cat_id not in", values, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdBetween(Integer value1, Integer value2) {
+            addCriterion("book_cat_id between", value1, value2, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andBookCatIdNotBetween(Integer value1, Integer value2) {
+            addCriterion("book_cat_id not between", value1, value2, "bookCatId");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeIsNull() {
+            addCriterion("last_update_time is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeIsNotNull() {
+            addCriterion("last_update_time is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeEqualTo(Date value) {
+            addCriterion("last_update_time =", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeNotEqualTo(Date value) {
+            addCriterion("last_update_time <>", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeGreaterThan(Date value) {
+            addCriterion("last_update_time >", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeGreaterThanOrEqualTo(Date value) {
+            addCriterion("last_update_time >=", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeLessThan(Date value) {
+            addCriterion("last_update_time <", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeLessThanOrEqualTo(Date value) {
+            addCriterion("last_update_time <=", value, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeIn(List<Date> values) {
+            addCriterion("last_update_time in", values, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeNotIn(List<Date> values) {
+            addCriterion("last_update_time not in", values, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeBetween(Date value1, Date value2) {
+            addCriterion("last_update_time between", value1, value2, "lastUpdateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andLastUpdateTimeNotBetween(Date value1, Date value2) {
+            addCriterion("last_update_time not between", value1, value2, "lastUpdateTime");
+            return (Criteria) this;
+        }
+    }
+
+    public static class Criteria extends GeneratedCriteria {
+
+        protected Criteria() {
+            super();
+        }
+    }
+
+    public static class Criterion {
+        private String condition;
+
+        private Object value;
+
+        private Object secondValue;
+
+        private boolean noValue;
+
+        private boolean singleValue;
+
+        private boolean betweenValue;
+
+        private boolean listValue;
+
+        private String typeHandler;
+
+        public String getCondition() {
+            return condition;
+        }
+
+        public Object getValue() {
+            return value;
+        }
+
+        public Object getSecondValue() {
+            return secondValue;
+        }
+
+        public boolean isNoValue() {
+            return noValue;
+        }
+
+        public boolean isSingleValue() {
+            return singleValue;
+        }
+
+        public boolean isBetweenValue() {
+            return betweenValue;
+        }
+
+        public boolean isListValue() {
+            return listValue;
+        }
+
+        public String getTypeHandler() {
+            return typeHandler;
+        }
+
+        protected Criterion(String condition) {
+            super();
+            this.condition = condition;
+            this.typeHandler = null;
+            this.noValue = true;
+        }
+
+        protected Criterion(String condition, Object value, String typeHandler) {
+            super();
+            this.condition = condition;
+            this.value = value;
+            this.typeHandler = typeHandler;
+            if (value instanceof List<?>) {
+                this.listValue = true;
+            } else {
+                this.singleValue = true;
+            }
+        }
+
+        protected Criterion(String condition, Object value) {
+            this(condition, value, null);
+        }
+
+        protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
+            super();
+            this.condition = condition;
+            this.value = value;
+            this.secondValue = secondValue;
+            this.typeHandler = typeHandler;
+            this.betweenValue = true;
+        }
+
+        protected Criterion(String condition, Object value, Object secondValue) {
+            this(condition, value, secondValue, null);
+        }
+    }
+}

novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java

@@ -3,30 +3,20 @@ package xyz.zinglizingli.books.service;
 import com.github.pagehelper.PageHelper;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.codec.Charsets;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.DateUtils;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.Resource;
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import tk.mybatis.orderbyhelper.OrderByHelper;
 import xyz.zinglizingli.books.core.constant.CacheKeyConstans;
 import xyz.zinglizingli.books.core.enums.PicSaveType;
+import xyz.zinglizingli.books.core.utils.Constants;
 import xyz.zinglizingli.books.mapper.*;
 import xyz.zinglizingli.books.po.*;
-import xyz.zinglizingli.books.core.utils.Constants;
+import xyz.zinglizingli.common.cache.CommonCacheUtil;
 import xyz.zinglizingli.common.utils.FileUtil;
 import xyz.zinglizingli.common.utils.SpringUtil;
-import xyz.zinglizingli.common.utils.UUIDUtils;
-import xyz.zinglizingli.common.cache.CommonCacheUtil;
-import xyz.zinglizingli.common.utils.RestTemplateUtil;
 
-import java.io.*;
 import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -51,6 +41,8 @@ public class BookService {
 
     private final BookParseLogMapper bookParseLogMapper;
 
+    private final BookUpdateTimeLogMapper bookUpdateTimeLogMapper;
+
     private final CommonCacheUtil cacheUtil;
 
 
@@ -132,7 +124,6 @@ public class BookService {
                 bookService.insertIndexListAndContentList(newBookIndexList, newContentList);
             }
 
-            cacheUtil.del(CacheKeyConstans.NEWST_BOOK_LIST_KEY);
 
 
         }
@@ -255,7 +246,7 @@ public class BookService {
             example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(indexNum);
             List<BookContent> bookContents = bookContentMapper.selectByExample(example);
             content = bookContents.size() > 0 ? bookContents.get(0) : null;
-            cacheUtil.setObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum, content, 60 * 60 * 24);
+            cacheUtil.setObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum, content, 60 * 10);
         }
 
         return content;
@@ -450,7 +441,7 @@ public class BookService {
      * */
     public void addBookParseLog(String bookUrl, String bookName, Float score) {
         BookParseLogExample example = new BookParseLogExample();
-        example.createCriteria().andBookUrlEqualTo(bookUrl);
+        example.createCriteria().andBookUrlEqualTo(bookUrl).andCreateTimeGreaterThan(new Date(System.currentTimeMillis()-1000*60*60));
         if(bookParseLogMapper.countByExample(example)==0) {
             BookParseLog bookParseLog = new BookParseLog();
             bookParseLog.setBookUrl(bookUrl);
@@ -491,4 +482,39 @@ public class BookService {
         example.createCriteria().andBookNameEqualTo(bookName).andAuthorEqualTo(author);
         return bookMapper.countByExample(example)>0;
     }
+
+    /**
+     * 查询分类更新时间映射信息
+     * */
+    public Map<Integer, Date> queryLastUpdateTime() {
+        List<BookUpdateTimeLog> list = bookUpdateTimeLogMapper.selectByExample(new BookUpdateTimeLogExample());
+
+        return list.stream().collect(Collectors.toMap(BookUpdateTimeLog::getBookCatId, BookUpdateTimeLog::getLastUpdateTime,(key1, key2) -> key2));
+
+    }
+
+    /**
+     * 更新分类时间日志
+     * */
+    public void updateBookUpdateTimeLog(Map<Integer, Date> cat2Date) {
+        if(cat2Date.size()>0) {
+            Set<Map.Entry<Integer, Date>> entries = cat2Date.entrySet();
+            for(Map.Entry<Integer, Date> entry : entries){
+                BookUpdateTimeLogExample example = new BookUpdateTimeLogExample();
+                example.createCriteria().andBookCatIdEqualTo(entry.getKey());
+                BookUpdateTimeLog entity = new BookUpdateTimeLog();
+                entity.setLastUpdateTime(entry.getValue());
+                bookUpdateTimeLogMapper.updateByExampleSelective(entity,example);
+
+            }
+        }
+    }
+
+    /**
+     * 删除已经成功更新的解析日志
+     * */
+    public void deleteBookParseLog(Long id) {
+        bookParseLogMapper.deleteByPrimaryKey(id);
+
+    }
 }

novel-front/src/main/resources/application-crawl.yml

@@ -54,6 +54,23 @@ dingdian:
     catalog-url-pattern: <a\s+href="(/ddk\d+/all.html)">查看完整目录</a>
     catalog-pattern: <a\s+style=""\s+href="(/ddk\d+/\d+\.html)">([^/]+)</a>
 
+
+baishuzhai:
+  crawlsource:
+    index-url: https://m.baishuzhai.com
+    list-page-url: https://m.baishuzhai.com/sort/{0}/{1}.html
+    book-url-pattern: href="/(ibook/\d+/\d+)/"
+    score-pattern: <div\s+class="score">(\d+\.\d+)分</div>
+    book-name-pattern: <p class="title">([^/]+)</p>
+    author-pattern: 作者：([^/]+)<
+    status-pattern: 状态：([^/]+)</li>
+    cat-pattern: 类别：([^/]+)</li>
+    update-time-pattern: 更新：(\d+-\d+-\d+)</li>
+    pic-pattern: <img src="([^>]+)"\s+onerror="this.src=
+    intro-pattern: class="review">([^/]+)</p>
+    catalog-url-pattern: <a\s+href="(/ibook/\d+/\d+/all\.html)">查看完整目录</a>
+    catalog-pattern: <a\s+style=""\s+href="(/ibook/\d+/\d+/\d+\.html)">([^/]+)</a>
+
 biquge:
   crawlsource:
     index-url: http://m.biquge.info

novel-front/src/main/resources/application.yml

@@ -45,6 +45,19 @@ spring:
 
 
 
+
+
+  resources:
+    cache:
+      # 资源缓存时间，单位秒
+      period: 604800
+    # 开启gzip压缩
+    chain:
+      gzipped: true
+      # 启用缓存
+      cache: true
+
+
 #  mvc:
 #    static-path-pattern: /static/** #设定静态文件路径，js,css等
 mybatis:
@@ -65,14 +78,16 @@ index:
 mysql:
   charset: utf8mb4
 
-#爬取小说数据的最低评分
 books:
+  #爬取小说数据的最低评分
   lowestScore: 6.0
+  #小说的更新间隔（分）
+  updatePeriod: 1
 
-#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说  更多网站解析中，敬请期待
+#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔,3:顶点小说 ，4：百书斋 更多网站解析中，敬请期待
 crawl:
   website:
-    type: 2
+    type: 4
 
 
 

novel-front/src/main/resources/mybatis/generatorConfig.xml

@@ -42,7 +42,7 @@
 			<property name="enableSubPackages" value="false" />
 		</javaClientGenerator>
 
-		<table  tableName="book_parse_log"/>
+		<table  tableName="book_update_time_log"/>
 		
 		<!-- 指定数据库表 -->
 		<!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->

novel-front/src/main/resources/mybatis/mapping/BookUpdateTimeLogMapper.xml

@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="xyz.zinglizingli.books.mapper.BookUpdateTimeLogMapper" >
+  <resultMap id="BaseResultMap" type="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    <id column="id" property="id" jdbcType="INTEGER" />
+    <result column="book_cat_id" property="bookCatId" jdbcType="INTEGER" />
+    <result column="last_update_time" property="lastUpdateTime" jdbcType="TIMESTAMP" />
+  </resultMap>
+  <sql id="Example_Where_Clause" >
+    <where >
+      <foreach collection="oredCriteria" item="criteria" separator="or" >
+        <if test="criteria.valid" >
+          <trim prefix="(" suffix=")" prefixOverrides="and" >
+            <foreach collection="criteria.criteria" item="criterion" >
+              <choose >
+                <when test="criterion.noValue" >
+                  and ${criterion.condition}
+                </when>
+                <when test="criterion.singleValue" >
+                  and ${criterion.condition} #{criterion.value}
+                </when>
+                <when test="criterion.betweenValue" >
+                  and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
+                </when>
+                <when test="criterion.listValue" >
+                  and ${criterion.condition}
+                  <foreach collection="criterion.value" item="listItem" open="(" close=")" separator="," >
+                    #{listItem}
+                  </foreach>
+                </when>
+              </choose>
+            </foreach>
+          </trim>
+        </if>
+      </foreach>
+    </where>
+  </sql>
+  <sql id="Update_By_Example_Where_Clause" >
+    <where >
+      <foreach collection="example.oredCriteria" item="criteria" separator="or" >
+        <if test="criteria.valid" >
+          <trim prefix="(" suffix=")" prefixOverrides="and" >
+            <foreach collection="criteria.criteria" item="criterion" >
+              <choose >
+                <when test="criterion.noValue" >
+                  and ${criterion.condition}
+                </when>
+                <when test="criterion.singleValue" >
+                  and ${criterion.condition} #{criterion.value}
+                </when>
+                <when test="criterion.betweenValue" >
+                  and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
+                </when>
+                <when test="criterion.listValue" >
+                  and ${criterion.condition}
+                  <foreach collection="criterion.value" item="listItem" open="(" close=")" separator="," >
+                    #{listItem}
+                  </foreach>
+                </when>
+              </choose>
+            </foreach>
+          </trim>
+        </if>
+      </foreach>
+    </where>
+  </sql>
+  <sql id="Base_Column_List" >
+    id, book_cat_id, last_update_time
+  </sql>
+  <select id="selectByExample" resultMap="BaseResultMap" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLogExample" >
+    select
+    <if test="distinct" >
+      distinct
+    </if>
+    <include refid="Base_Column_List" />
+    from book_update_time_log
+    <if test="_parameter != null" >
+      <include refid="Example_Where_Clause" />
+    </if>
+    <if test="orderByClause != null" >
+      order by ${orderByClause}
+    </if>
+  </select>
+  <select id="selectByPrimaryKey" resultMap="BaseResultMap" parameterType="java.lang.Integer" >
+    select 
+    <include refid="Base_Column_List" />
+    from book_update_time_log
+    where id = #{id,jdbcType=INTEGER}
+  </select>
+  <delete id="deleteByPrimaryKey" parameterType="java.lang.Integer" >
+    delete from book_update_time_log
+    where id = #{id,jdbcType=INTEGER}
+  </delete>
+  <delete id="deleteByExample" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLogExample" >
+    delete from book_update_time_log
+    <if test="_parameter != null" >
+      <include refid="Example_Where_Clause" />
+    </if>
+  </delete>
+  <insert id="insert" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    insert into book_update_time_log (id, book_cat_id, last_update_time
+      )
+    values (#{id,jdbcType=INTEGER}, #{bookCatId,jdbcType=INTEGER}, #{lastUpdateTime,jdbcType=TIMESTAMP}
+      )
+  </insert>
+  <insert id="insertSelective" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    insert into book_update_time_log
+    <trim prefix="(" suffix=")" suffixOverrides="," >
+      <if test="id != null" >
+        id,
+      </if>
+      <if test="bookCatId != null" >
+        book_cat_id,
+      </if>
+      <if test="lastUpdateTime != null" >
+        last_update_time,
+      </if>
+    </trim>
+    <trim prefix="values (" suffix=")" suffixOverrides="," >
+      <if test="id != null" >
+        #{id,jdbcType=INTEGER},
+      </if>
+      <if test="bookCatId != null" >
+        #{bookCatId,jdbcType=INTEGER},
+      </if>
+      <if test="lastUpdateTime != null" >
+        #{lastUpdateTime,jdbcType=TIMESTAMP},
+      </if>
+    </trim>
+  </insert>
+  <select id="countByExample" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLogExample" resultType="java.lang.Integer" >
+    select count(*) from book_update_time_log
+    <if test="_parameter != null" >
+      <include refid="Example_Where_Clause" />
+    </if>
+  </select>
+  <update id="updateByExampleSelective" parameterType="map" >
+    update book_update_time_log
+    <set >
+      <if test="record.id != null" >
+        id = #{record.id,jdbcType=INTEGER},
+      </if>
+      <if test="record.bookCatId != null" >
+        book_cat_id = #{record.bookCatId,jdbcType=INTEGER},
+      </if>
+      <if test="record.lastUpdateTime != null" >
+        last_update_time = #{record.lastUpdateTime,jdbcType=TIMESTAMP},
+      </if>
+    </set>
+    <if test="_parameter != null" >
+      <include refid="Update_By_Example_Where_Clause" />
+    </if>
+  </update>
+  <update id="updateByExample" parameterType="map" >
+    update book_update_time_log
+    set id = #{record.id,jdbcType=INTEGER},
+      book_cat_id = #{record.bookCatId,jdbcType=INTEGER},
+      last_update_time = #{record.lastUpdateTime,jdbcType=TIMESTAMP}
+    <if test="_parameter != null" >
+      <include refid="Update_By_Example_Where_Clause" />
+    </if>
+  </update>
+  <update id="updateByPrimaryKeySelective" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    update book_update_time_log
+    <set >
+      <if test="bookCatId != null" >
+        book_cat_id = #{bookCatId,jdbcType=INTEGER},
+      </if>
+      <if test="lastUpdateTime != null" >
+        last_update_time = #{lastUpdateTime,jdbcType=TIMESTAMP},
+      </if>
+    </set>
+    where id = #{id,jdbcType=INTEGER}
+  </update>
+  <update id="updateByPrimaryKey" parameterType="xyz.zinglizingli.books.po.BookUpdateTimeLog" >
+    update book_update_time_log
+    set book_cat_id = #{bookCatId,jdbcType=INTEGER},
+      last_update_time = #{lastUpdateTime,jdbcType=TIMESTAMP}
+    where id = #{id,jdbcType=INTEGER}
+  </update>
+</mapper>

novel-front/src/main/resources/templates/books/book_content.html

@@ -19,17 +19,6 @@
     <div th:include="common/css :: css"></div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-
-
-    </script>
     <script type="text/javascript">
         function reinitIframe(){
 
@@ -477,16 +466,6 @@
 </script>
 <script>
     (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
         $("#content").css("min-height",($(window).height()-60)+"px");
 

novel-front/src/main/resources/templates/books/book_detail.html

@@ -115,30 +115,6 @@
 
     </style>
 
-    <script>
-
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function(){
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
     <script src="/js/wap_collect.js"></script>
 
 </head>

novel-front/src/main/resources/templates/books/book_index.html

@@ -39,29 +39,6 @@
     </div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function(){
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
 
 </head>
 <body>

novel-front/src/main/resources/templates/books/book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -236,18 +227,6 @@
 
 </script>
 <script>
-    (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 
 
     function toMyCollect(){

novel-front/src/main/resources/templates/books/index_v1.html

@@ -46,15 +46,6 @@
 
     </style>
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
 </head>
 <body>
@@ -247,17 +238,6 @@
 <script src="/js/wap_collect.js"></script>
 <script>
     lazyload();
-    (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        } else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 </script>
 
 

novel-front/src/main/resources/templates/books/index_v2.html

@@ -46,15 +46,6 @@
 
     </style>
 
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
 </head>
 <body>
@@ -293,17 +284,6 @@
 <script src="/js/wap_collect.js"></script>
 <script>
     lazyload();
-    (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        } else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 </script>
 
 

novel-front/src/main/resources/templates/books/mh_book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -247,16 +238,6 @@
 </script>
 <script>
     (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
 
         var sortCat = $("#softCat").val();

novel-front/src/main/resources/templates/books/soft_book_search.html

@@ -15,15 +15,6 @@
     <div th:include="common/css :: css"></div>
 
     </div>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
 
     <style type="text/css">
 
@@ -272,16 +263,6 @@
 </script>
 <script>
     (function () {
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
 
 
         var sortCat = $("#softCat").val();

novel-front/src/main/resources/templates/index.html

@@ -50,29 +50,6 @@
     </style>
 
     <script language="javascript" type="text/javascript" src="/js/wap_collect.js"></script>
-    <script>
-        var _hmt = _hmt || [];
-        (function () {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?b3a84b2ec6cc52dd088d735565b49644";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-    </script>
-    <script>
-        (function () {
-            var bp = document.createElement('script');
-            var curProtocol = window.location.protocol.split(':')[0];
-            if (curProtocol === 'https') {
-                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-            }
-            else {
-                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-            }
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(bp, s);
-        })();
-    </script>
     <style id="tsbrowser_video_independent_player_style" type="text/css">
         [tsbrowser_force_max_size] {
             width: 100% !important;

novel-front/src/main/resources/templates/user/login.html

@@ -10,17 +10,6 @@
     <div th:include="common/css :: css"></div>
 
 
-    <script>
-        var _hmt = _hmt || [];
-        (function() {
-            var hm = document.createElement("script");
-            hm.src = "https://hm.baidu.com/hm.js?2cf01edbc2b27cd3a143e17948167d77";
-            var s = document.getElementsByTagName("script")[0];
-            s.parentNode.insertBefore(hm, s);
-        })();
-
-
-    </script>
 
 </head>
 <body id="read" >
@@ -107,18 +96,6 @@
     });
 </script>
 <script>
-    (function(){
-        var bp = document.createElement('script');
-        var curProtocol = window.location.protocol.split(':')[0];
-        if (curProtocol === 'https') {
-            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
-        }
-        else {
-            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
-        }
-        var s = document.getElementsByTagName("script")[0];
-        s.parentNode.insertBefore(bp, s);
-    })();
 
 
 

script/crawlbook/application.yml

@@ -1,14 +1,15 @@
 server: {port: 8083}
 spring:
-  datasource: {url: 'jdbc:mysql://47.106.243.172:3306/novel?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai',
+  datasource: {url: 'jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai',
-    username: novel, password: novel!8888}
+    username: root, password: test123456}
 mybatis:
   mapper-locations: classpath:mybatis/mapping/*.xml
   type-aliases-package: xyz.zinglizingli.books.po
   configuration: {log-impl: org.apache.ibatis.logging.stdout.StdOutImpl}
 mysql: {charset: utf8mb4}
-books: {lowestScore: '9.0'}
+books: {lowestScore: 9.0}
 crawl:
-  website: {type: '2'}
+  website: {type: '4'}
   soft-novel: '0'
   manhua: '0'
+logging: {config: 'classpath:logback-boot.xml'}

sql/2020-01-17.sql

@@ -0,0 +1,21 @@
+
+DROP TABLE IF EXISTS `book_update_time_log`;
+CREATE TABLE `book_update_time_log` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `book_cat_id` int(11) NOT NULL,
+  `last_update_time` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_key_catid` (`book_cat_id`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4;
+
+
+INSERT INTO `book_update_time_log` VALUES ('1', '1', NOW());
+INSERT INTO `book_update_time_log` VALUES ('2', '2', NOW());
+INSERT INTO `book_update_time_log` VALUES ('3', '3', NOW());
+INSERT INTO `book_update_time_log` VALUES ('4', '4', NOW());
+INSERT INTO `book_update_time_log` VALUES ('5', '5', NOW());
+INSERT INTO `book_update_time_log` VALUES ('6', '6', NOW());
+INSERT INTO `book_update_time_log` VALUES ('7', '7', NOW());
+
+
+ALTER  TABLE  book_parse_log  drop  INDEX  uq_key_bookurl;

sql/books.sql

@@ -1024,3 +1024,21 @@ CREATE TABLE `book_parse_log` (
   UNIQUE KEY `uq_key_bookurl` (`book_url`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
 
+DROP TABLE IF EXISTS `book_update_time_log`;
+CREATE TABLE `book_update_time_log` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `book_cat_id` int(11) NOT NULL,
+  `last_update_time` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_key_catid` (`book_cat_id`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4;
+
+INSERT INTO `book_update_time_log` VALUES ('1', '1', NOW());
+INSERT INTO `book_update_time_log` VALUES ('2', '2', NOW());
+INSERT INTO `book_update_time_log` VALUES ('3', '3', NOW());
+INSERT INTO `book_update_time_log` VALUES ('4', '4', NOW());
+INSERT INTO `book_update_time_log` VALUES ('5', '5', NOW());
+INSERT INTO `book_update_time_log` VALUES ('6', '6', NOW());
+INSERT INTO `book_update_time_log` VALUES ('7', '7', NOW());
+
+ALTER  TABLE  book_parse_log  drop  INDEX  uq_key_bookurl;