mirror of
https://github.com/201206030/novel.git
synced 2025-06-13 11:18:30 +00:00
阿里巴巴规范改造
This commit is contained in:
xiongxiaoyang
parent
32847bc874
commit
289f1e1670
@ -5,105 +5,4 @@
|
||||
<configuration />
|
||||
</facet>
|
||||
</component>
|
||||
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
|
||||
<output url="file://$MODULE_DIR$/target/classes" />
|
||||
<output-test url="file://$MODULE_DIR$/target/test-classes" />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/target" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-autoconfigure:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-logging:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.2.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-to-slf4j:2.10.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.10.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.25" level="project" />
|
||||
<orderEntry type="library" name="Maven: javax.annotation:javax.annotation-api:1.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.yaml:snakeyaml:1.19" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-json:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.9.5" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.9.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.9.5" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.5" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.9.5" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-parameter-names:2.9.5" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-tomcat:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-core:8.5.29" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-el:8.5.29" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-websocket:8.5.29" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.hibernate.validator:hibernate-validator:6.0.9.Final" level="project" />
|
||||
<orderEntry type="library" name="Maven: javax.validation:validation-api:2.0.1.Final" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.jboss.logging:jboss-logging:3.3.2.Final" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.fasterxml:classmate:1.3.4" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-web:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-beans:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-webmvc:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-aop:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-expression:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-starter-test:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-test:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-test-autoconfigure:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: com.jayway.jsonpath:json-path:2.4.0" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: net.minidev:json-smart:2.3" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: net.minidev:accessors-smart:1.2" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.ow2.asm:asm:5.0.4" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.assertj:assertj-core:3.9.1" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-core:2.15.0" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: net.bytebuddy:byte-buddy:1.7.11" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: net.bytebuddy:byte-buddy-agent:1.7.11" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.6" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-library:1.3" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.skyscreamer:jsonassert:1.5.0" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: com.vaadin.external.google:android-json:0.0.20131108.vaadin1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-core:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-jcl:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.springframework:spring-test:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.xmlunit:xmlunit-core:2.5.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-cache:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-context:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-context-support:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.sf.ehcache:ehcache:2.10.4" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-thymeleaf:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.thymeleaf:thymeleaf-spring5:3.0.9.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.thymeleaf:thymeleaf:3.0.9.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.attoparser:attoparser:2.0.4.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.unbescape:unbescape:1.1.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.thymeleaf.extras:thymeleaf-extras-java8time:3.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.github.pagehelper:pagehelper-spring-boot-starter:1.2.5" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.github.pagehelper:pagehelper-spring-boot-autoconfigure:1.2.5" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.github.pagehelper:pagehelper:5.1.4" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.github.jsqlparser:jsqlparser:1.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.cuisongliu:orderbyhelper-spring-boot-starter:1.0.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.cuisongliu:orderbyhelper-spring-boot-autoconfigure:1.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: tk.mybatis:orderby-helper:0.0.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: mysql:mysql-connector-java:8.0.11" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.google.protobuf:protobuf-java:2.6.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.mybatis.spring.boot:mybatis-spring-boot-starter:1.3.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jdbc:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.zaxxer:HikariCP:2.7.8" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-jdbc:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-tx:5.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.mybatis.spring.boot:mybatis-spring-boot-autoconfigure:1.3.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.mybatis:mybatis:3.4.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.mybatis:mybatis-spring:1.3.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.10" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.11" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-mail:2.0.1.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.sun.mail:javax.mail:1.6.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.commons:commons-text:1.4" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.7" level="project" />
|
||||
</component>
|
||||
</module>
|
||||
@ -15,7 +15,6 @@
|
||||
<description>小说精品楼-前台web网站</description>
|
||||
|
||||
<properties>
|
||||
<!--开始没加这三个，报错Fatal error compiling: 无效的标记: -parameters ->-->
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<java.version>1.8</java.version>
|
||||
@ -69,6 +68,11 @@
|
||||
<version>1.3.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore -->
|
||||
<dependency>
|
||||
|
||||
@ -35,11 +35,6 @@ public interface BookMapper {
|
||||
|
||||
void addVisitCount(@Param("bookId") Long bookId);
|
||||
|
||||
Book queryRandomBook();
|
||||
|
||||
Book queryNewstBook(Set<Long> sendIds);
|
||||
|
||||
List<String> queryNewstBookIdList();
|
||||
|
||||
List<String> queryEndBookIdList();
|
||||
|
||||
|
||||
@ -1,20 +1,19 @@
|
||||
package xyz.zinglizingli.books.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.github.pagehelper.PageHelper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.Charsets;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.http.client.utils.DateUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import tk.mybatis.orderbyhelper.OrderByHelper;
|
||||
import xyz.zinglizingli.books.constant.CacheKeyConstans;
|
||||
import xyz.zinglizingli.books.mapper.*;
|
||||
@ -23,33 +22,31 @@ import xyz.zinglizingli.books.util.UUIDUtils;
|
||||
import xyz.zinglizingli.common.cache.CommonCacheUtil;
|
||||
import xyz.zinglizingli.common.utils.RestTemplateUtil;
|
||||
|
||||
import java.io.*;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* @author XXY
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class BookService {
|
||||
|
||||
@Autowired
|
||||
private BookMapper bookMapper;
|
||||
private final BookMapper bookMapper;
|
||||
|
||||
@Autowired
|
||||
private BookIndexMapper bookIndexMapper;
|
||||
private final BookIndexMapper bookIndexMapper;
|
||||
|
||||
@Autowired
|
||||
private BookContentMapper bookContentMapper;
|
||||
private final BookContentMapper bookContentMapper;
|
||||
|
||||
@Autowired
|
||||
private ScreenBulletMapper screenBulletMapper;
|
||||
private final ScreenBulletMapper screenBulletMapper;
|
||||
|
||||
@Autowired
|
||||
private UserRefBookMapper userRefBookMapper;
|
||||
private final UserRefBookMapper userRefBookMapper;
|
||||
|
||||
@Autowired
|
||||
private CommonCacheUtil cacheUtil;
|
||||
|
||||
RestTemplate isoRestTemplate = RestTemplateUtil.getInstance("iso-8859-1");
|
||||
private final CommonCacheUtil cacheUtil;
|
||||
|
||||
@Value("${pic.save.type}")
|
||||
private Byte picSaveType;
|
||||
@ -58,16 +55,11 @@ public class BookService {
|
||||
private String picSavePath;
|
||||
|
||||
|
||||
private Logger log = LoggerFactory.getLogger(BookService.class);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 保存章节目录和内容
|
||||
* */
|
||||
public void saveBookAndIndexAndContent(Book book, List<BookIndex> bookIndex, List<BookContent> bookContent){
|
||||
//一次最多只允许插入20条记录,否则影响服务器响应,如果没有插入所有更新，则更新时间设为昨天
|
||||
/*if(bookIndex.size()>100){
|
||||
book.setUpdateTime(new Date(book.getUpdateTime().getTime()-1000*60*60*24));
|
||||
}
|
||||
*/
|
||||
|
||||
boolean isUpdate = false;
|
||||
Long bookId = -1L;
|
||||
@ -81,12 +73,12 @@ public class BookService {
|
||||
bookId = books.get(0).getId();
|
||||
book.setId(bookId);
|
||||
String picSrc = book.getPicUrl();
|
||||
if(picSaveType == 2 && org.apache.commons.lang3.StringUtils.isNotBlank(picSrc)){
|
||||
if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
|
||||
ResponseEntity<Resource> resEntity = isoRestTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
|
||||
InputStream input = resEntity.getBody().getInputStream();
|
||||
ResponseEntity<Resource> resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
|
||||
InputStream input = Objects.requireNonNull(resEntity.getBody()).getInputStream();
|
||||
Date currentDate = new Date();
|
||||
picSrc = "/localPic/" + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd")
|
||||
+ UUIDUtils.getUUID32()
|
||||
@ -127,12 +119,7 @@ public class BookService {
|
||||
}
|
||||
|
||||
if (bookId >= 0) {
|
||||
//查询目录已存在数量
|
||||
/* BookIndexExample bookIndexExample = new BookIndexExample();
|
||||
bookIndexExample.createCriteria().andBookIdEqualTo(bookId);
|
||||
int indexCount = bookIndexMapper.countByExample(bookIndexExample);*/
|
||||
|
||||
BookIndex lastIndex = null;
|
||||
List<BookIndex> newBookIndexList = new ArrayList<>();
|
||||
List<BookContent> newContentList = new ArrayList<>();
|
||||
for (int i = 0; i < bookIndex.size(); i++) {
|
||||
@ -141,11 +128,9 @@ public class BookService {
|
||||
BookIndex bookIndexItem = bookIndex.get(i);
|
||||
bookIndexItem.setBookId(bookId);
|
||||
bookContentItem.setBookId(bookId);
|
||||
//bookContentItem.setIndexId(bookIndexItem.getId());暂时使用bookId和IndexNum查询content
|
||||
bookContentItem.setIndexNum(bookIndexItem.getIndexNum());
|
||||
newBookIndexList.add(bookIndexItem);
|
||||
newContentList.add(bookContentItem);
|
||||
lastIndex = bookIndexItem;
|
||||
}
|
||||
//一次最多只允许插入20条记录,否则影响服务器响应
|
||||
if (isUpdate && i % 20 == 0 && newBookIndexList.size() > 0) {
|
||||
@ -174,15 +159,21 @@ public class BookService {
|
||||
|
||||
}
|
||||
|
||||
@Transactional
|
||||
/**
|
||||
* 批量插入章节目录表和章节内容表
|
||||
* */
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void insertIndexListAndContentList(List<BookIndex> newBookIndexList, List<BookContent> newContentList) {
|
||||
bookIndexMapper.insertBatch(newBookIndexList);
|
||||
bookContentMapper.insertBatch(newContentList);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 生成随机访问次数
|
||||
* */
|
||||
private Long generateVisiteCount(Float score) {
|
||||
int baseNum = (int) (Math.pow(score * 10, (int) (score - 5)) / 2);
|
||||
int baseNum = (int)(score * 100);
|
||||
return Long.parseLong(baseNum + new Random().nextInt(1000) + "");
|
||||
}
|
||||
|
||||
@ -202,12 +193,13 @@ public class BookService {
|
||||
OrderByHelper.orderBy(sortBy + " " + sort);
|
||||
}
|
||||
|
||||
List<Book> books = bookMapper.search(userId, ids, keyword, catId, softCat, softTag, bookStatus);
|
||||
|
||||
return books;
|
||||
return bookMapper.search(userId, ids, keyword, catId, softCat, softTag, bookStatus);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取分类名
|
||||
* */
|
||||
public String getCatNameById(Integer catid) {
|
||||
String catName = "其他";
|
||||
|
||||
@ -257,11 +249,17 @@ public class BookService {
|
||||
return catName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询书籍的基础数据
|
||||
* */
|
||||
public Book queryBaseInfo(Long bookId) {
|
||||
|
||||
return bookMapper.selectByPrimaryKey(bookId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询最新更新的书籍列表
|
||||
* */
|
||||
public List<BookIndex> queryNewIndexList(Long bookId) {
|
||||
PageHelper.startPage(1, 15);
|
||||
BookIndexExample example = new BookIndexExample();
|
||||
@ -271,6 +269,9 @@ public class BookService {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询书籍目录列表
|
||||
* */
|
||||
public List<BookIndex> queryAllIndexList(Long bookId) {
|
||||
BookIndexExample example = new BookIndexExample();
|
||||
example.createCriteria().andBookIdEqualTo(bookId);
|
||||
@ -278,6 +279,9 @@ public class BookService {
|
||||
return bookIndexMapper.selectByExample(example);
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询书籍章节内容
|
||||
* */
|
||||
public BookContent queryBookContent(Long bookId, Integer indexNum) {
|
||||
BookContent content = (BookContent) cacheUtil.getObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum);
|
||||
if (content == null) {
|
||||
@ -285,11 +289,6 @@ public class BookService {
|
||||
example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(indexNum);
|
||||
List<BookContent> bookContents = bookContentMapper.selectByExample(example);
|
||||
content = bookContents.size() > 0 ? bookContents.get(0) : null;
|
||||
/*try {
|
||||
content.setContent(chargeBookContent(content.getContent()));
|
||||
} catch (IOException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}*/
|
||||
cacheUtil.setObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum, content, 60 * 60 * 24);
|
||||
}
|
||||
|
||||
@ -297,6 +296,9 @@ public class BookService {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 增加访问次数
|
||||
* */
|
||||
public void addVisitCount(Long bookId, String userId, Integer indexNum) {
|
||||
|
||||
bookMapper.addVisitCount(bookId);
|
||||
@ -307,13 +309,23 @@ public class BookService {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询章节名
|
||||
* */
|
||||
public String queryIndexNameByBookIdAndIndexNum(Long bookId, Integer indexNum) {
|
||||
|
||||
BookIndexExample example = new BookIndexExample();
|
||||
example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(indexNum);
|
||||
return bookIndexMapper.selectByExample(example).get(0).getIndexName();
|
||||
List<BookIndex> indexList = bookIndexMapper.selectByExample(example);
|
||||
if(indexList != null && indexList.size() > 0 ) {
|
||||
return indexList.get(0).getIndexName();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询最大和最小章节号
|
||||
* */
|
||||
public List<Integer> queryMaxAndMinIndexNum(Long bookId) {
|
||||
List<Integer> result = new ArrayList<>();
|
||||
BookIndexExample example = new BookIndexExample();
|
||||
@ -330,7 +342,7 @@ public class BookService {
|
||||
/**
|
||||
* 查询该书籍目录数量
|
||||
*/
|
||||
public List<Integer> queryIndexCountByBookNameAndBAuthor(String bookName, String author) {
|
||||
public List<Integer> queryIndexCountByBookNameAndAuthor(String bookName, String author) {
|
||||
List<Integer> result = new ArrayList<>();
|
||||
BookExample example = new BookExample();
|
||||
example.createCriteria().andBookNameEqualTo(bookName).andAuthorEqualTo(author);
|
||||
@ -353,33 +365,11 @@ public class BookService {
|
||||
|
||||
}
|
||||
|
||||
public Book queryRandomBook() {
|
||||
|
||||
return bookMapper.queryRandomBook();
|
||||
}
|
||||
|
||||
public Map<String, Object> queryNewstBook() {
|
||||
final String SENDIDS = "sendWeiboIds";
|
||||
Set<Long> sendIds = (Set<Long>) cacheUtil.getObject(SENDIDS);
|
||||
if (sendIds == null) {
|
||||
sendIds = new HashSet<>();
|
||||
}
|
||||
String newstIndexName = null;
|
||||
Book book = null;
|
||||
book = bookMapper.queryNewstBook(sendIds);
|
||||
Map<String, Object> data = new HashMap<>();
|
||||
if (book != null && book.getId() != null) {
|
||||
newstIndexName = bookIndexMapper.queryNewstIndexName(book.getId());
|
||||
if (!StringUtils.isEmpty(newstIndexName)) {
|
||||
sendIds.add(book.getId());
|
||||
cacheUtil.setObject(SENDIDS, sendIds, 60 * 60 * 24 * 2);
|
||||
data.put("book", book);
|
||||
data.put("newstIndexName", newstIndexName);
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询轻小说分类名
|
||||
* */
|
||||
public String getSoftCatNameById(Integer softCat) {
|
||||
String catName = "其他";
|
||||
|
||||
@ -434,6 +424,9 @@ public class BookService {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询漫画分类名
|
||||
* */
|
||||
public String getMhCatNameById(Integer softCat) {
|
||||
String catName = "其他";
|
||||
|
||||
@ -456,6 +449,9 @@ public class BookService {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存弹幕
|
||||
* */
|
||||
public void sendBullet(Long contentId, String bullet) {
|
||||
|
||||
ScreenBullet screenBullet = new ScreenBullet();
|
||||
@ -466,6 +462,9 @@ public class BookService {
|
||||
screenBulletMapper.insertSelective(screenBullet);
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询弹幕
|
||||
* */
|
||||
public List<ScreenBullet> queryBullet(Long contentId) {
|
||||
|
||||
ScreenBulletExample example = new ScreenBulletExample();
|
||||
@ -475,19 +474,19 @@ public class BookService {
|
||||
return screenBulletMapper.selectByExample(example);
|
||||
}
|
||||
|
||||
public String queryIndexList(Long bookId, int count) {
|
||||
|
||||
BookIndexExample example = new BookIndexExample();
|
||||
example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(count);
|
||||
return bookIndexMapper.selectByExample(example).get(0).getIndexName();
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询章节内容
|
||||
* */
|
||||
public String queryContentList(Long bookId, int count) {
|
||||
BookContentExample example = new BookContentExample();
|
||||
example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(count);
|
||||
return bookContentMapper.selectByExample(example).get(0).getContent();
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询章节数
|
||||
* */
|
||||
public int countIndex(Long bookId) {
|
||||
BookIndexExample example = new BookIndexExample();
|
||||
example.createCriteria().andBookIdEqualTo(bookId);
|
||||
@ -495,6 +494,9 @@ public class BookService {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 查询完本书籍
|
||||
* */
|
||||
public List<String> queryEndBookIdList() {
|
||||
return bookMapper.queryEndBookIdList();
|
||||
}
|
||||
@ -503,6 +505,9 @@ public class BookService {
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 查询前一章节和后一章节号
|
||||
* */
|
||||
public List<Integer> queryPreAndNextIndexNum(Long bookId, Integer indexNum) {
|
||||
List<Integer> result = new ArrayList<>();
|
||||
BookIndexExample example = new BookIndexExample();
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
package xyz.zinglizingli.books.service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.mail.SimpleMailMessage;
|
||||
@ -14,36 +14,52 @@ import javax.mail.MessagingException;
|
||||
import javax.mail.internet.InternetAddress;
|
||||
import javax.mail.internet.MimeMessage;
|
||||
import java.io.File;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
|
||||
/**
|
||||
* @author XXY
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class MailService {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(MailService.class);
|
||||
|
||||
/**
|
||||
* 使用@Value注入application.properties中指定的用户名
|
||||
* */
|
||||
@Value("${spring.mail.username}")
|
||||
//使用@Value注入application.properties中指定的用户名
|
||||
private String from;
|
||||
|
||||
String nickName = "精品小说楼";
|
||||
private String nickName = "精品小说楼";
|
||||
|
||||
@Autowired
|
||||
//用于发送文件
|
||||
private JavaMailSender mailSender;
|
||||
/**
|
||||
* 用于发送文件
|
||||
* */
|
||||
private final JavaMailSender mailSender;
|
||||
|
||||
|
||||
/**
|
||||
* 发送简单邮件
|
||||
* */
|
||||
public void sendSimpleMail(String to, String subject, String content) {
|
||||
|
||||
SimpleMailMessage message = new SimpleMailMessage();
|
||||
message.setTo(to);//收信人
|
||||
message.setSubject(subject);//主题
|
||||
message.setText(content);//内容
|
||||
message.setFrom(from);//发信人
|
||||
//收信人
|
||||
message.setTo(to);
|
||||
//主题
|
||||
message.setSubject(subject);
|
||||
//内容
|
||||
message.setText(content);
|
||||
//发信人
|
||||
message.setFrom(from);
|
||||
|
||||
mailSender.send(message);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 发送html邮件
|
||||
* */
|
||||
public void sendHtmlMail(String to, String subject, String content){
|
||||
|
||||
logger.info("发送HTML邮件开始：{},{},{}", to, subject, content);
|
||||
@ -57,7 +73,8 @@ public class MailService {
|
||||
helper.setFrom(new InternetAddress(from, nickName, "UTF-8"));
|
||||
helper.setTo(to);
|
||||
helper.setSubject(subject);
|
||||
helper.setText(content, true);//true代表支持html
|
||||
//true代表支持html
|
||||
helper.setText(content, true);
|
||||
mailSender.send(message);
|
||||
logger.info("发送HTMLto"+to+"邮件成功");
|
||||
} catch (Exception e) {
|
||||
@ -65,6 +82,9 @@ public class MailService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送带附件的邮件
|
||||
* */
|
||||
public void sendAttachmentMail(String to, String subject, String content, String filePath) {
|
||||
|
||||
logger.info("发送带附件邮件开始：{},{},{},{}", to, subject, content, filePath);
|
||||
@ -80,7 +100,8 @@ public class MailService {
|
||||
helper.setText(content, true);
|
||||
FileSystemResource file = new FileSystemResource(new File(filePath));
|
||||
String fileName = file.getFilename();
|
||||
helper.addAttachment(fileName, file);//添加附件，可多次调用该方法添加多个附件
|
||||
//添加附件，可多次调用该方法添加多个附件
|
||||
helper.addAttachment(fileName, file);
|
||||
mailSender.send(message);
|
||||
logger.info("发送带附件邮件成功");
|
||||
} catch (MessagingException e) {
|
||||
@ -103,7 +124,8 @@ public class MailService {
|
||||
helper.setSubject(subject);
|
||||
helper.setText(content, true);
|
||||
FileSystemResource res = new FileSystemResource(new File(rscPath));
|
||||
helper.addInline(rscId, res);//重复使用添加多个图片
|
||||
//重复使用添加多个图片
|
||||
helper.addInline(rscId, res);
|
||||
mailSender.send(message);
|
||||
logger.info("发送带图片邮件成功");
|
||||
} catch (Exception e) {
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
package xyz.zinglizingli.books.service;
|
||||
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
import xyz.zinglizingli.books.mapper.UserMapper;
|
||||
import xyz.zinglizingli.books.mapper.UserRefBookMapper;
|
||||
@ -13,27 +13,38 @@ import xyz.zinglizingli.books.util.MD5Util;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author XXY
|
||||
*/
|
||||
@RequiredArgsConstructor
|
||||
@Service
|
||||
public class UserService {
|
||||
|
||||
@Autowired
|
||||
private UserMapper userMapper;
|
||||
private final UserMapper userMapper;
|
||||
|
||||
@Autowired
|
||||
private UserRefBookMapper userRefBookMapper;
|
||||
private final UserRefBookMapper userRefBookMapper;
|
||||
|
||||
|
||||
/**
|
||||
* 判断登录名是否存在
|
||||
* */
|
||||
public boolean isExistLoginName(String loginName) {
|
||||
UserExample example = new UserExample();
|
||||
example.createCriteria().andLoginNameEqualTo(loginName);
|
||||
return userMapper.countByExample(example)>0?true:false;
|
||||
return userMapper.countByExample(example)>0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 注册
|
||||
* */
|
||||
public void regist(User user) {
|
||||
user.setPassword(MD5Util.MD5Encode(user.getPassword(),"utf-8"));
|
||||
userMapper.insertSelective(user);
|
||||
}
|
||||
|
||||
/**
|
||||
* 登陆
|
||||
* */
|
||||
public void login(User user) {
|
||||
UserExample example = new UserExample();
|
||||
example.createCriteria().andLoginNameEqualTo(user.getLoginName())
|
||||
@ -48,6 +59,9 @@ public class UserService {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 加入书架
|
||||
* */
|
||||
public void addToCollect(Long bookId, long userId) {
|
||||
UserRefBook userRefBook = new UserRefBook();
|
||||
userRefBook.setBookId(bookId);
|
||||
@ -59,20 +73,29 @@ public class UserService {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断是否加入书架
|
||||
* */
|
||||
public boolean isCollect(Long bookId, long userId) {
|
||||
|
||||
UserRefBookExample example = new UserRefBookExample();
|
||||
example.createCriteria().andBookIdEqualTo(bookId).andUserIdEqualTo(userId);
|
||||
return userRefBookMapper.countByExample(example)>0?true:false;
|
||||
return userRefBookMapper.countByExample(example)>0;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消加入书架
|
||||
* */
|
||||
public void cancelToCollect(Long bookId, long userId) {
|
||||
UserRefBookExample example = new UserRefBookExample();
|
||||
example.createCriteria().andBookIdEqualTo(bookId).andUserIdEqualTo(userId);
|
||||
userRefBookMapper.deleteByExample(example);
|
||||
}
|
||||
|
||||
/**
|
||||
* 加入或取消书架
|
||||
* */
|
||||
public void collectOrCancelBook(Long userid, Long bookId) {
|
||||
|
||||
boolean collect = isCollect(bookId, userid);
|
||||
@ -84,6 +107,9 @@ public class UserService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询用户章节阅读记录
|
||||
* */
|
||||
public Integer queryBookIndexNumber(String userId, Long bookId) {
|
||||
return userRefBookMapper.queryBookIndexNumber(userId,bookId);
|
||||
}
|
||||
|
||||
@ -3,6 +3,9 @@ package xyz.zinglizingli.books.util;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
/**
|
||||
* @author XXY
|
||||
*/
|
||||
public class ExcutorUtils {
|
||||
|
||||
private static ExecutorService fixedThreadPool;
|
||||
|
||||
@ -2,10 +2,13 @@ package xyz.zinglizingli.books.web;
|
||||
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import xyz.zinglizingli.books.po.Book;
|
||||
import xyz.zinglizingli.books.po.BookContent;
|
||||
import xyz.zinglizingli.books.po.BookIndex;
|
||||
@ -13,22 +16,26 @@ import xyz.zinglizingli.books.service.BookService;
|
||||
import xyz.zinglizingli.books.vo.BookVO;
|
||||
import xyz.zinglizingli.common.cache.CommonCacheUtil;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* api接口
|
||||
* @author XXY
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("api/book")
|
||||
@RequiredArgsConstructor
|
||||
public class ApiBookController {
|
||||
|
||||
|
||||
@Autowired
|
||||
private BookService bookService;
|
||||
private final BookService bookService;
|
||||
|
||||
@Autowired
|
||||
private CommonCacheUtil commonCacheUtil;
|
||||
private final CommonCacheUtil commonCacheUtil;
|
||||
|
||||
|
||||
/**
|
||||
* 首页热门书籍查询接口
|
||||
* */
|
||||
@RequestMapping("hotBook")
|
||||
public List<Book> hotBooks () {
|
||||
//查询热点数据
|
||||
@ -36,6 +43,9 @@ public class ApiBookController {
|
||||
return hotBooks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 首页最新书籍查询接口
|
||||
* */
|
||||
@RequestMapping("newstBook")
|
||||
public List<Book> newstBook() {
|
||||
//查询最近更新数据
|
||||
@ -44,6 +54,9 @@ public class ApiBookController {
|
||||
return newBooks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 书籍搜索接口
|
||||
* */
|
||||
@RequestMapping("search")
|
||||
public Map<String,Object> search(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize,
|
||||
@RequestParam(value = "keyword", required = false) String keyword,
|
||||
@ -51,15 +64,14 @@ public class ApiBookController {
|
||||
@RequestParam(value = "catId", required = false) Integer catId,
|
||||
@RequestParam(value = "historyBookIds", required = false) String ids,
|
||||
@RequestParam(value = "token", required = false) String token,
|
||||
@RequestParam(value = "sortBy", defaultValue = "update_time") String sortBy, @RequestParam(value = "sort", defaultValue = "DESC") String sort,
|
||||
HttpServletRequest req, HttpServletResponse resp) {
|
||||
@RequestParam(value = "sortBy", defaultValue = "update_time") String sortBy, @RequestParam(value = "sort", defaultValue = "DESC") String sort
|
||||
) {
|
||||
|
||||
Map<String,Object> modelMap = new HashMap<>();
|
||||
String userId = null;
|
||||
String titleType = "最近更新";
|
||||
if (catId != null) {
|
||||
titleType = bookService.getCatNameById(catId);
|
||||
;
|
||||
} else if (keyword != null) {
|
||||
titleType = "搜索";
|
||||
} else if ("score".equals(sortBy)) {
|
||||
@ -114,6 +126,9 @@ public class ApiBookController {
|
||||
return modelMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* 书籍详情信息查询接口
|
||||
* */
|
||||
@RequestMapping("{bookId}.html")
|
||||
public Map<String,Object> detail(@PathVariable("bookId") Long bookId) {
|
||||
Map<String,Object> modelMap = new HashMap<>();
|
||||
@ -131,6 +146,9 @@ public class ApiBookController {
|
||||
return modelMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* 书籍目录查询接口
|
||||
* */
|
||||
@RequestMapping("{bookId}/index.html")
|
||||
public Map<String,Object> bookIndex(@PathVariable("bookId") Long bookId) {
|
||||
Map<String,Object> modelMap = new HashMap<>();
|
||||
@ -142,6 +160,9 @@ public class ApiBookController {
|
||||
return modelMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* 书籍章节内容查询接口
|
||||
* */
|
||||
@RequestMapping("{bookId}/{indexNum}.html")
|
||||
public Map<String,Object> bookContent(@PathVariable("bookId") Long bookId, @PathVariable("indexNum") Integer indexNum) {
|
||||
Map<String,Object> modelMap = new HashMap<>();
|
||||
@ -161,7 +182,7 @@ public class ApiBookController {
|
||||
String indexName;
|
||||
if(bookContent==null) {
|
||||
bookContent = new BookContent();
|
||||
bookContent.setId(-1l);
|
||||
bookContent.setId(-1L);
|
||||
bookContent.setBookId(bookId);
|
||||
bookContent.setIndexNum(indexNum);
|
||||
bookContent.setContent("正在手打中，请稍等片刻，内容更新后，需要重新刷新页面，才能获取最新更新");
|
||||
@ -176,13 +197,6 @@ public class ApiBookController {
|
||||
return modelMap;
|
||||
}
|
||||
|
||||
/*@RequestMapping("addVisit")
|
||||
public String addVisit(@RequestParam("bookId") Long bookId) {
|
||||
|
||||
bookService.addVisitCount(bookId, userId, indexNum);
|
||||
|
||||
return "ok";
|
||||
}*/
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -2,9 +2,9 @@ package xyz.zinglizingli.books.web;
|
||||
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.text.StringEscapeUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.ui.ModelMap;
|
||||
import org.springframework.util.StringUtils;
|
||||
@ -26,24 +26,30 @@ import javax.servlet.http.HttpServletResponse;
|
||||
import javax.servlet.http.HttpSession;
|
||||
import java.io.OutputStream;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 小说Controller
|
||||
* @author 11797
|
||||
*/
|
||||
@Controller
|
||||
@RequestMapping("book")
|
||||
@RequiredArgsConstructor
|
||||
public class BookController {
|
||||
|
||||
|
||||
@Autowired
|
||||
private BookService bookService;
|
||||
private final BookService bookService;
|
||||
|
||||
@Autowired
|
||||
private UserService userService;
|
||||
private final UserService userService;
|
||||
|
||||
@Autowired
|
||||
private CommonCacheUtil commonCacheUtil;
|
||||
private final CommonCacheUtil commonCacheUtil;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 精品小说搜索页
|
||||
* */
|
||||
@RequestMapping("search")
|
||||
public String search(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize,
|
||||
@RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "catId", required = false) Integer catId,
|
||||
@ -120,6 +126,9 @@ public class BookController {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 轻小说搜索页
|
||||
* */
|
||||
@RequestMapping("searchSoftBook.html")
|
||||
public String searchSoftBook(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize,
|
||||
@RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "catId", defaultValue = "8") Integer catId,
|
||||
@ -155,6 +164,9 @@ public class BookController {
|
||||
return "books/soft_book_search";
|
||||
}
|
||||
|
||||
/**
|
||||
* 漫画搜索页
|
||||
* */
|
||||
@RequestMapping("searchMhBook.html")
|
||||
public String searchMhBook(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize,
|
||||
@RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "catId", defaultValue = "9") Integer catId,
|
||||
@ -190,6 +202,9 @@ public class BookController {
|
||||
return "books/mh_book_search";
|
||||
}
|
||||
|
||||
/**
|
||||
* 书籍详情页
|
||||
* */
|
||||
@RequestMapping("{bookId}.html")
|
||||
public String detail(@PathVariable("bookId") Long bookId, @RequestParam(value = "token",required = false)String token, ModelMap modelMap) {
|
||||
String userId = commonCacheUtil.get(token);
|
||||
@ -229,6 +244,9 @@ public class BookController {
|
||||
return "books/book_detail";
|
||||
}
|
||||
|
||||
/**
|
||||
* 书籍目录页
|
||||
* */
|
||||
@RequestMapping("{bookId}/index.html")
|
||||
public String bookIndex(@PathVariable("bookId") Long bookId, ModelMap modelMap) {
|
||||
List<BookIndex> indexList = bookService.queryAllIndexList(bookId);
|
||||
@ -239,13 +257,17 @@ public class BookController {
|
||||
return "books/book_index";
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 书籍内容页
|
||||
* */
|
||||
@RequestMapping("{bookId}/{indexNum}.html")
|
||||
public String bookContent(@PathVariable("bookId") Long bookId, @PathVariable("indexNum") Integer indexNum, ModelMap modelMap) {
|
||||
BookContent bookContent = bookService.queryBookContent(bookId, indexNum);
|
||||
String indexName;
|
||||
if (bookContent == null) {
|
||||
bookContent = new BookContent();
|
||||
bookContent.setId(-1l);
|
||||
bookContent.setId(-1L);
|
||||
bookContent.setBookId(bookId);
|
||||
bookContent.setIndexNum(indexNum);
|
||||
bookContent.setContent("正在手打中，请稍等片刻，内容更新后，需要重新刷新页面，才能获取最新更新");
|
||||
@ -271,6 +293,9 @@ public class BookController {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 增加访问次数
|
||||
* */
|
||||
@RequestMapping("addVisit")
|
||||
@ResponseBody
|
||||
public String addVisit(@RequestParam("bookId") Long bookId,@RequestParam(value = "indexNum",defaultValue = "0") Integer indexNum,@RequestParam(value = "token",defaultValue = "") String token) {
|
||||
@ -281,6 +306,9 @@ public class BookController {
|
||||
return "ok";
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送弹幕
|
||||
* */
|
||||
@RequestMapping("sendBullet")
|
||||
@ResponseBody
|
||||
public Map<String, Object> sendBullet(@RequestParam("contentId") Long contentId, @RequestParam("bullet") String bullet) {
|
||||
@ -291,6 +319,9 @@ public class BookController {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询是否正在下载
|
||||
* */
|
||||
@RequestMapping("queryIsDownloading")
|
||||
@ResponseBody
|
||||
public Map<String, Object> queryIsDownloading(HttpSession session) {
|
||||
@ -304,6 +335,9 @@ public class BookController {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 查询弹幕
|
||||
* */
|
||||
@RequestMapping("queryBullet")
|
||||
@ResponseBody
|
||||
public List<ScreenBullet> queryBullet(@RequestParam("contentId") Long contentId) {
|
||||
@ -325,27 +359,30 @@ public class BookController {
|
||||
OutputStream out = resp.getOutputStream();
|
||||
//设置响应头，对文件进行url编码
|
||||
bookName = URLEncoder.encode(bookName, "UTF-8");
|
||||
resp.setContentType("application/octet-stream");//解决手机端不能下载附件的问题
|
||||
//解决手机端不能下载附件的问题
|
||||
resp.setContentType("application/octet-stream");
|
||||
resp.setHeader("Content-Disposition", "attachment;filename=" + bookName + ".txt");
|
||||
if (count > 0) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
String index = bookService.queryIndexList(bookId, i);
|
||||
String content = bookService.queryContentList(bookId, i);
|
||||
out.write(index.getBytes("utf-8"));
|
||||
out.write("\n".getBytes("utf-8"));
|
||||
content = content.replaceAll("<br\\s*/*>", "\r\n");
|
||||
content = content.replaceAll(" ", " ");
|
||||
content = content.replaceAll("<a[^>]*>", "");
|
||||
content = content.replaceAll("</a>", "");
|
||||
content = content.replaceAll("<div[^>]*>", "");
|
||||
content = content.replaceAll("</div>", "");
|
||||
content = content.replaceAll("<p[^>]*>[^<]*<a[^>]*>[^<]*</a>\\s*</p>", "");
|
||||
content = content.replaceAll("<p[^>]*>", "");
|
||||
content = content.replaceAll("</p>", "\r\n");
|
||||
out.write(content.getBytes("utf-8"));
|
||||
out.write("\r\n".getBytes("utf-8"));
|
||||
out.write("\r\n".getBytes("utf-8"));
|
||||
out.flush();
|
||||
String index = bookService.queryIndexNameByBookIdAndIndexNum(bookId, i);
|
||||
if(index != null) {
|
||||
String content = bookService.queryContentList(bookId, i);
|
||||
out.write(index.getBytes(StandardCharsets.UTF_8));
|
||||
out.write("\n".getBytes(StandardCharsets.UTF_8));
|
||||
content = content.replaceAll("<br\\s*/*>", "\r\n");
|
||||
content = content.replaceAll(" ", " ");
|
||||
content = content.replaceAll("<a[^>]*>", "");
|
||||
content = content.replaceAll("</a>", "");
|
||||
content = content.replaceAll("<div[^>]*>", "");
|
||||
content = content.replaceAll("</div>", "");
|
||||
content = content.replaceAll("<p[^>]*>[^<]*<a[^>]*>[^<]*</a>\\s*</p>", "");
|
||||
content = content.replaceAll("<p[^>]*>", "");
|
||||
content = content.replaceAll("</p>", "\r\n");
|
||||
out.write(content.getBytes(StandardCharsets.UTF_8));
|
||||
out.write("\r\n".getBytes(StandardCharsets.UTF_8));
|
||||
out.write("\r\n".getBytes(StandardCharsets.UTF_8));
|
||||
out.flush();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package xyz.zinglizingli.books.web;
|
||||
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.ui.ModelMap;
|
||||
@ -15,21 +16,23 @@ import xyz.zinglizingli.common.cache.CommonCacheUtil;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author XXY
|
||||
*/
|
||||
@Controller
|
||||
@RequestMapping("user")
|
||||
@RequiredArgsConstructor
|
||||
public class UserController {
|
||||
|
||||
|
||||
@Autowired
|
||||
private UserService userService;
|
||||
private final UserService userService;
|
||||
|
||||
@Autowired
|
||||
private BookService bookService;
|
||||
|
||||
@Autowired
|
||||
private CommonCacheUtil commonCacheUtil;
|
||||
private final CommonCacheUtil commonCacheUtil;
|
||||
|
||||
|
||||
/**
|
||||
* 登陆页面
|
||||
* */
|
||||
@RequestMapping("login.html")
|
||||
public String login(Long bookId, ModelMap modelMap) {
|
||||
modelMap.put("bookId", bookId);
|
||||
@ -37,10 +40,13 @@ public class UserController {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 登陆或注册
|
||||
* */
|
||||
@RequestMapping("loginOrRegist")
|
||||
@ResponseBody
|
||||
public Map<String, Object> loginOrRegist(User user,Long bookId) {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
Map<String, Object> result = new HashMap<>(2);
|
||||
//查询用户名是否存在
|
||||
boolean isExistLoginName = userService.isExistLoginName(user.getLoginName());
|
||||
String token = null;
|
||||
@ -77,10 +83,13 @@ public class UserController {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 登陆状态查询
|
||||
* */
|
||||
@RequestMapping("isLogin")
|
||||
@ResponseBody
|
||||
public Map<String, Object> isLogin(String token) {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
Map<String, Object> result = new HashMap<>(2);
|
||||
String userId = commonCacheUtil.get(token);
|
||||
if(StringUtils.isEmpty(userId)){
|
||||
result.put("code", -1);
|
||||
@ -93,6 +102,9 @@ public class UserController {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 加入书架
|
||||
* */
|
||||
@RequestMapping("addToCollect")
|
||||
@ResponseBody
|
||||
public Map<String, Object> addToCollect(Long bookId,String token) {
|
||||
@ -109,10 +121,13 @@ public class UserController {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 撤下书架
|
||||
* */
|
||||
@RequestMapping("cancelToCollect")
|
||||
@ResponseBody
|
||||
public Map<String, Object> cancelToCollect(Long bookId,String token) {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
Map<String, Object> result = new HashMap<>(2);
|
||||
String userId = commonCacheUtil.get(token);
|
||||
if(StringUtils.isEmpty(userId)){
|
||||
result.put("code", -1);
|
||||
@ -125,10 +140,13 @@ public class UserController {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断是否加入书架
|
||||
* */
|
||||
@RequestMapping("isCollect")
|
||||
@ResponseBody
|
||||
public Map<String, Object> isCollect(Long bookId,String token) {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
Map<String, Object> result = new HashMap<>(2);
|
||||
String userId = commonCacheUtil.get(token);
|
||||
if(StringUtils.isEmpty(userId)){
|
||||
result.put("code", -1);
|
||||
|
||||
@ -5,6 +5,7 @@ import org.springframework.boot.web.server.ErrorPageRegistrar;
|
||||
import org.springframework.boot.web.server.ErrorPageRegistry;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import xyz.zinglizingli.common.utils.Constants;
|
||||
|
||||
/**
|
||||
*定义配置类
|
||||
@ -15,8 +16,8 @@ public class ErrorConfig implements ErrorPageRegistrar {
|
||||
@Override
|
||||
public void registerErrorPages(ErrorPageRegistry registry) {
|
||||
ErrorPage[] errorPages = new ErrorPage[2];
|
||||
errorPages[0] = new ErrorPage(HttpStatus.NOT_FOUND, "/book/index.html");
|
||||
errorPages[1] = new ErrorPage(HttpStatus.INTERNAL_SERVER_ERROR, "/mang.html");
|
||||
errorPages[0] = new ErrorPage(HttpStatus.NOT_FOUND, Constants.NOT_FOUND_PATH);
|
||||
errorPages[1] = new ErrorPage(HttpStatus.INTERNAL_SERVER_ERROR, Constants.SERVER_ERROR_PATH);
|
||||
|
||||
registry.addErrorPages(errorPages);
|
||||
}
|
||||
|
||||
@ -4,7 +4,7 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.web.servlet.FilterRegistrationBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import xyz.zinglizingli.common.filter.SearchFilter;
|
||||
import xyz.zinglizingli.common.filter.BookFilter;
|
||||
|
||||
@Configuration
|
||||
public class FilterConfig{
|
||||
@ -13,9 +13,9 @@ public class FilterConfig{
|
||||
private String picSavePath;
|
||||
|
||||
@Bean
|
||||
public FilterRegistrationBean filterRegist() {
|
||||
FilterRegistrationBean frBean = new FilterRegistrationBean();
|
||||
frBean.setFilter(new SearchFilter());
|
||||
public FilterRegistrationBean<BookFilter> filterRegist() {
|
||||
FilterRegistrationBean<BookFilter> frBean = new FilterRegistrationBean<>();
|
||||
frBean.setFilter(new BookFilter());
|
||||
frBean.addUrlPatterns("/*");
|
||||
frBean.addInitParameter("picSavePath",picSavePath);
|
||||
return frBean;
|
||||
|
||||
@ -6,6 +6,10 @@ import org.springframework.stereotype.Component;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 首页推荐书籍配置
|
||||
* @author XXY
|
||||
* */
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "index")
|
||||
public class IndexRecBooksConfig {
|
||||
|
||||
@ -0,0 +1,222 @@
|
||||
package xyz.zinglizingli.common.filter;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.Charsets;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.HttpClientErrorException;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import xyz.zinglizingli.common.utils.Constants;
|
||||
import xyz.zinglizingli.common.utils.RestTemplateUtil;
|
||||
|
||||
import javax.servlet.*;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 书籍内容过滤器
|
||||
* @author xxy
|
||||
*/
|
||||
@Slf4j
|
||||
public class BookFilter implements Filter {
|
||||
|
||||
|
||||
/**
|
||||
* 图片后缀集合
|
||||
* */
|
||||
private final List<String> PIC_POSTFIX_LIST = Arrays.asList("jpg","pcx","emf","gif","bmp","tga","jpeg","tif","png","rle");
|
||||
|
||||
/**
|
||||
* 本地图片保存路径前缀
|
||||
* */
|
||||
private String picSavePath;
|
||||
|
||||
|
||||
@Override
|
||||
public void init(FilterConfig filterConfig){
|
||||
picSavePath = filterConfig.getInitParameter("picSavePath");
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain){
|
||||
HttpServletRequest req = (HttpServletRequest) servletRequest;
|
||||
HttpServletResponse resp = (HttpServletResponse) servletResponse;
|
||||
String requestUrl = req.getRequestURL().toString();
|
||||
String requestUri = req.getRequestURI();
|
||||
|
||||
String forObject;
|
||||
|
||||
try {
|
||||
|
||||
//本地图片
|
||||
if (requestUri.contains(Constants.LOCAL_PIC_PREFIX)) {
|
||||
OutputStream out = resp.getOutputStream();
|
||||
InputStream input = new FileInputStream(new File(picSavePath + requestUri));
|
||||
byte[] b = new byte[4096];
|
||||
for (int n; (n = input.read(b)) != -1; ) {
|
||||
out.write(b, 0, n);
|
||||
}
|
||||
input.close();
|
||||
out.close();
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
//非漫画访问
|
||||
if (!requestUrl.contains(Constants.CARTOON_CONTENT_PREFIX)) {
|
||||
filterChain.doFilter(req, resp);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
String method = req.getMethod();
|
||||
|
||||
//请求html/css/js等文件
|
||||
RestTemplate restTemplate = RestTemplateUtil.getInstance(Charsets.UTF_8);
|
||||
|
||||
resp.setCharacterEncoding("utf-8");
|
||||
|
||||
String realPath = Constants.CRAWL_CARTOON_URL_PREFIX + requestUri.substring(Constants.CARTOON_CONTENT_PREFIX.length());
|
||||
|
||||
|
||||
if (HttpMethod.GET.name().equals(method)) {
|
||||
//漫画GET请求
|
||||
String postFix = requestUri.substring(requestUri.lastIndexOf(".") + 1);
|
||||
if (PIC_POSTFIX_LIST.contains(postFix)) {
|
||||
// 请求图片
|
||||
restTemplate = RestTemplateUtil.getInstance(Charsets.ISO_8859_1);
|
||||
resp.setContentType("image/apng");
|
||||
}
|
||||
if (requestUri.startsWith(Constants.CARTOON_PIC_PREFIX)) {
|
||||
//漫画图片处理
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add("Referer", Constants.CRAWL_CARTOON_URL_PREFIX);
|
||||
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
|
||||
realPath = Constants.CRAWL_CARTOON_PIC_URL_PREFIX + requestUri.substring(Constants.CARTOON_PIC_PREFIX.length());
|
||||
ResponseEntity<Resource> resEntity = restTemplate.exchange(realPath, HttpMethod.GET, requestEntity, Resource.class);
|
||||
InputStream input = Objects.requireNonNull(resEntity.getBody()).getInputStream();
|
||||
OutputStream out = resp.getOutputStream();
|
||||
byte[] b = new byte[4096];
|
||||
for (int n; (n = input.read(b)) != -1; ) {
|
||||
out.write(b, 0, n);
|
||||
}
|
||||
input.close();
|
||||
out.close();
|
||||
return;
|
||||
|
||||
|
||||
} else if (requestUri.startsWith(Constants.CARTOON_STATIC_PREFIX)) {
|
||||
//漫画静态文件处理
|
||||
realPath = Constants.CRAWL_CARTOON_STATIC_URL_PREFIX + requestUri.substring(Constants.CARTOON_STATIC_PREFIX.length());
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(realPath, String.class);
|
||||
forObject = forEntity.getBody();
|
||||
assert forObject != null;
|
||||
forObject = forObject.replaceAll(Constants.CRAWL_CARTOON_PIC_URL_PREFIX, Constants.CARTOON_PIC_PREFIX);
|
||||
|
||||
} else {
|
||||
//其他漫画内容处理
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(realPath, String.class);
|
||||
forObject = forEntity.getBody();
|
||||
|
||||
assert forObject != null;
|
||||
|
||||
//漫画内容过滤
|
||||
forObject = forObject.replace(Constants.CRAWL_CARTOON_URL_PREFIX+"js/ad/ad_12.js", "")
|
||||
.replace(Constants.CRAWL_CARTOON_URL_PREFIX+"js/ad/ad_13.js", "")
|
||||
.replace(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX+"ocomic/js/dmzjMhFinally-new.js", "")
|
||||
.replace(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX+"ocomic/js/dmzjMhFinally-new.js", "")
|
||||
.replace(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX+"module/js/float_code.js", "")
|
||||
.replaceAll(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX+"public/js/dmzj-land-2015.6.js", "")
|
||||
.replaceAll("<script type=\"text/javascript\">var cnzz_protocol =[^<]+</script>", "")
|
||||
.replaceAll("globalNav.js", "")
|
||||
.replaceAll("TSB.js", "")
|
||||
.replaceAll(Constants.CRAWL_CARTOON_PIC_URL_PREFIX, Constants.CARTOON_PIC_PREFIX)
|
||||
.replaceAll(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX, Constants.CARTOON_STATIC_PREFIX)
|
||||
+ "<script>var browser={\n" +
|
||||
" versions:function(){\n" +
|
||||
" var u = window.navigator.userAgent;\n" +
|
||||
" return {\n" +
|
||||
" trident: u.indexOf('Trident') > -1, //IE内核\n" +
|
||||
" presto: u.indexOf('Presto') > -1, //opera内核\n" +
|
||||
" webKit: u.indexOf('AppleWebKit') > -1, //苹果、谷歌内核\n" +
|
||||
" gecko: u.indexOf('Gecko') > -1 && u.indexOf('KHTML') == -1, //火狐内核\n" +
|
||||
" mobile: !!u.match(/AppleWebKit.*Mobile.*/)||!!u.match(/AppleWebKit/), //是否为移动终端\n" +
|
||||
" ios: !!u.match(/\\(i[^;]+;( U;)? CPU.+Mac OS X/), //ios终端\n" +
|
||||
" android: u.indexOf('Android') > -1 || u.indexOf('Linux') > -1, //android终端或者uc浏览器\n" +
|
||||
" iPhone: u.indexOf('iPhone') > -1 || u.indexOf('Mac') > -1, //是否为iPhone或者安卓QQ浏览器\n" +
|
||||
" iPad: u.indexOf('iPad') > -1, //是否为iPad\n" +
|
||||
" webApp: u.indexOf('Safari') == -1 ,//是否为web应用程序，没有头部与底部\n" +
|
||||
" weixin: u.indexOf('MicroMessenger') == -1 //是否为微信浏览器\n" +
|
||||
" };\n" +
|
||||
" }()\n" +
|
||||
"};" +
|
||||
"</script>"
|
||||
+ "<script>$(function(){$(\"#app_manhua\").remove();" +
|
||||
"$('.btmBtnBox').remove();$('.red_box').remove()" +
|
||||
";$('.mainNav').remove();$('.wrap_last_head').remove();$('.wrap_last_mid').remove()" +
|
||||
";$('.comic_gd').remove();$('.comic_last').remove();$('.side_bar').remove()" +
|
||||
";$('.point_wrap').remove();$('.show').remove();$('.light').remove()" +
|
||||
";$('#sidePublic').remove();$('.side_public').remove()" +
|
||||
";$('.foot-detail').remove();$('#float_nav_type').remove()})</script>";
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
//漫画POST请求
|
||||
|
||||
|
||||
Map<String, String[]> oldParameterMap = req.getParameterMap();
|
||||
Set<Map.Entry<String, String[]>> entries = oldParameterMap.entrySet();
|
||||
Map<String, String> newParameterMap = new HashMap<>(entries.size());
|
||||
for (Map.Entry<String, String[]> entry : entries) {
|
||||
newParameterMap.put(entry.getKey(), entry.getValue()[0]);
|
||||
}
|
||||
|
||||
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
|
||||
map.setAll(newParameterMap);
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
|
||||
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
|
||||
forObject = restTemplate.postForEntity(realPath, request, String.class).getBody();
|
||||
assert forObject != null;
|
||||
forObject = forObject.replaceAll(Constants.CARTOON_CONTENT_PREFIX, Constants.CRAWL_CARTOON_URL_PREFIX)
|
||||
;
|
||||
|
||||
}
|
||||
|
||||
|
||||
} catch (RuntimeException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
if (e instanceof HttpClientErrorException && (((HttpClientErrorException) e).getStatusCode() == HttpStatus.NOT_FOUND)) {
|
||||
//404
|
||||
resp.sendRedirect(Constants.NOT_FOUND_PATH);
|
||||
return;
|
||||
} else {
|
||||
//500
|
||||
req.getRequestDispatcher(Constants.SERVER_ERROR_PATH).forward(servletRequest, servletResponse);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
resp.getWriter().print(forObject);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,541 +0,0 @@
|
||||
package xyz.zinglizingli.common.filter;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.HttpClientErrorException;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import xyz.zinglizingli.common.cache.CommonCacheUtil;
|
||||
import xyz.zinglizingli.common.utils.RestTemplateUtil;
|
||||
import xyz.zinglizingli.common.utils.SpringUtil;
|
||||
|
||||
import javax.servlet.*;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.*;
|
||||
import java.net.URLDecoder;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class SearchFilter implements Filter {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SearchFilter.class);
|
||||
|
||||
private CommonCacheUtil cacheUtil;
|
||||
|
||||
private static List<String> picPostFix;
|
||||
|
||||
private static List<String> localFileFix;
|
||||
|
||||
private static List<String> staticFileFix;
|
||||
|
||||
private static List<String> noteURI;
|
||||
|
||||
private RestTemplate restTemplate;
|
||||
|
||||
private String picSavePath;
|
||||
|
||||
|
||||
|
||||
private final String SUANWEI_BOOK_REGEX = "<a\\s+href=\"/(\\d+_\\d+)/\">";
|
||||
private final String SUANWEI_BOOK_HTML_REGEX = "/\\d+_\\d+\\.html";
|
||||
|
||||
private final String XIYANGYANG_BOOK_REGEX = "<a\\s+href=\"/(\\d+_\\d+)/\">";
|
||||
private final String XIYANGYANG_BOOK_HTML_REGEX = "/\\d+_\\d+\\.html";
|
||||
|
||||
|
||||
@Override
|
||||
public void init(FilterConfig filterConfig) throws ServletException {
|
||||
picSavePath = filterConfig.getInitParameter("picSavePath");
|
||||
picPostFix = new ArrayList<>();
|
||||
picPostFix.add("jpg");
|
||||
picPostFix.add("pcx");
|
||||
picPostFix.add("emf");
|
||||
picPostFix.add("gif");
|
||||
picPostFix.add("bmp");
|
||||
picPostFix.add("tga");
|
||||
picPostFix.add("jpeg");
|
||||
picPostFix.add("tif");
|
||||
picPostFix.add("png");
|
||||
picPostFix.add("rle");
|
||||
localFileFix = new ArrayList<>();
|
||||
localFileFix.add("IMG_1470.JPG");
|
||||
localFileFix.add("baidu_verify_Ep8xaWQJAI.html");
|
||||
localFileFix.add("baidu_verify_L6sR9GjEtg.html");
|
||||
localFileFix.add("shenma-site-verification.txt");
|
||||
localFileFix.add("favicon.ico");
|
||||
localFileFix.add("headerbg.jpg");
|
||||
localFileFix.add("mang.png");
|
||||
localFileFix.add("HotBook.apk");
|
||||
localFileFix.add("wap_collect.js");
|
||||
localFileFix.add("note_1.html");
|
||||
localFileFix.add("note_2.html");
|
||||
localFileFix.add("note_3.html");
|
||||
localFileFix.add("note_4.html");
|
||||
staticFileFix = new ArrayList<>();
|
||||
staticFileFix.add("jpg");
|
||||
staticFileFix.add("gif");
|
||||
staticFileFix.add("bmp");
|
||||
staticFileFix.add("jpeg");
|
||||
staticFileFix.add("png");
|
||||
staticFileFix.add("js");
|
||||
staticFileFix.add("css");
|
||||
noteURI = new ArrayList<>();
|
||||
noteURI.add("/html/note_1.html");
|
||||
noteURI.add("/html/note_2.html");
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
|
||||
String forObject = null;
|
||||
HttpServletRequest req = (HttpServletRequest) servletRequest;
|
||||
HttpServletResponse resp = (HttpServletResponse) servletResponse;
|
||||
String requestURL = req.getRequestURL().toString();
|
||||
String requestURI = req.getRequestURI();
|
||||
|
||||
try {
|
||||
|
||||
if(requestURI.contains("/localPic/")){
|
||||
OutputStream out = resp.getOutputStream();
|
||||
InputStream input = new FileInputStream(new File(picSavePath+requestURI));
|
||||
byte[] b = new byte[4096];
|
||||
for (int n; (n = input.read(b)) != -1;) {
|
||||
out.write(b, 0, n);
|
||||
}
|
||||
input.close();
|
||||
out.close();
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
if (!requestURL.contains("/manhua/")) {
|
||||
filterChain.doFilter(req, resp);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
final String method = req.getMethod();
|
||||
|
||||
|
||||
// 案例：充当客户端通过restTemplate请求网络数据，并充当服务端将数据返回给浏览器
|
||||
// 客户端请求数据：输入流（byte[]）==》字符串
|
||||
// 服务端响应数据：字符串 == 》 输出流（byte[]）
|
||||
|
||||
//默认方式：
|
||||
//RestTemplate restTemplate = new RestTemplate();
|
||||
// ①当返回的response-header的content-type属性有charset值时，
|
||||
// restTemplate的StringHttpMessageConverter会设置默认charset为content-type属性
|
||||
// charset值
|
||||
// StringHttpMessageConverter.setDefaultCharset(Charset.forName(charset));
|
||||
// ②当返回的response-header的content-type属性没有charset值时
|
||||
// restTemplate的StringHttpMessageConverter会使用默认的charset即ISO-8859-1
|
||||
|
||||
// 对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String）
|
||||
restTemplate = RestTemplateUtil.getInstance("utf-8");//请求html/css/js等文件
|
||||
// 对客户端响应返回的字符串（String）采用何种编码转换成输出流（byte[]）
|
||||
resp.setCharacterEncoding("utf-8");
|
||||
|
||||
/*//=====现在浏览器有编码自动识别功能，所以上面的代码没有加content-type的Header也没有问题==========
|
||||
//=====正确做法应该是下面代码片段1和代码片段2二选一==========
|
||||
|
||||
//===============================================代码片段1===============================
|
||||
// 对客户端响应返回的字符串（String）采用何种编码转换成输出流（byte[]）
|
||||
resp.setCharacterEncoding("utf-8");
|
||||
// 告诉浏览器对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String）显示
|
||||
resp.setHeader("content-type", "text/html;charset=utf-8");
|
||||
//===============================================代码片段1===============================
|
||||
|
||||
|
||||
//===============================================代码片段2===============================
|
||||
//对客户端响应返回的字符串（String）采用何种编码转换成输出流（byte[]）
|
||||
//并且告诉浏览器对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String）显示
|
||||
resp.setContentType("text/html;charset=utf-8");
|
||||
//===============================================代码片段2===============================
|
||||
*/
|
||||
|
||||
|
||||
if (HttpMethod.GET.name().equals(method)) {
|
||||
|
||||
|
||||
String fileName = requestURI.substring(requestURI.lastIndexOf("/") + 1);
|
||||
if (localFileFix.contains(fileName) || fileName.startsWith("9a4a540e-1759-4268-90fa-7fb652c3604a.")) {
|
||||
filterChain.doFilter(servletRequest, servletResponse);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
String queryString = req.getQueryString();
|
||||
if (queryString != null && queryString.length() > 0 && !queryString.contains("bsh_bid=")) {
|
||||
queryString = "?" + URLDecoder.decode(req.getQueryString());
|
||||
} else {
|
||||
queryString = "";
|
||||
}
|
||||
|
||||
|
||||
if (forObject == null) {
|
||||
String postFix = requestURI.substring(requestURI.lastIndexOf(".") + 1);
|
||||
if (picPostFix.contains(postFix)) {
|
||||
// 对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String）
|
||||
restTemplate = RestTemplateUtil.getInstance("iso-8859-1");//请求图片
|
||||
resp.setContentType("image/apng");
|
||||
}
|
||||
if (requestURI.startsWith("/manhua/images/")) {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add("Referer","https://www.dmzj.com");
|
||||
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
|
||||
String realUrl = "https://images.dmzj.com/" + requestURI.substring(15);
|
||||
ResponseEntity<Resource> resEntity = restTemplate.exchange(realUrl, HttpMethod.GET, requestEntity, Resource.class);
|
||||
InputStream input = resEntity.getBody().getInputStream();
|
||||
OutputStream out = resp.getOutputStream();
|
||||
byte[] b = new byte[4096];
|
||||
for (int n; (n = input.read(b)) != -1;) {
|
||||
out.write(b, 0, n);
|
||||
}
|
||||
input.close();
|
||||
out.close();
|
||||
return;
|
||||
|
||||
|
||||
// 对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String）
|
||||
|
||||
|
||||
|
||||
|
||||
} else if(requestURI.startsWith("/manhua/statics/")){
|
||||
String realPath = "https://static.dmzj.com/"+requestURI.substring(16);
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(realPath, String.class);
|
||||
forObject = forEntity.getBody();
|
||||
forObject = forObject.replaceAll("https://images.dmzj.com/", "/manhua/images/");
|
||||
|
||||
}else {
|
||||
String realPath = "https://www.dmzj.com/" + requestURI.substring(8);
|
||||
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(realPath, String.class);
|
||||
forObject = forEntity.getBody();
|
||||
|
||||
// forObject = new String(forObject.getBytes("ISO-8859-1"),"utf-8");
|
||||
forObject = forObject.replace("https://www.dmzj.com/js/ad/ad_12.js", "");
|
||||
forObject = forObject.replace("https://www.dmzj.com/js/ad/ad_13.js", "");
|
||||
forObject = forObject.replace("https://static.dmzj.com/ocomic/js/dmzjMhFinally-new.js", "");
|
||||
forObject = forObject.replace("https://static.dmzj.com/ocomic/js/dmzjMhFinally-new.js", "");
|
||||
forObject = forObject.replace("https://static.dmzj.com/module/js/float_code.js", "");
|
||||
forObject = forObject.replaceAll("<script type=\"text/javascript\">var cnzz_protocol =[^<]+</script>", "");
|
||||
forObject = forObject.replaceAll("https://static.dmzj.com/public/js/dmzj-land-2015.6.js", "");
|
||||
forObject = forObject.replaceAll("<script type=\"text/javascript\">var cnzz_protocol =[^<]+</script>", "");
|
||||
forObject = forObject.replaceAll("<script type=\"text/javascript\">var cnzz_protocol =[^<]+</script>", "");
|
||||
forObject = forObject.replaceAll("<script type=\"text/javascript\">var cnzz_protocol =[^<]+</script>", "");
|
||||
forObject = forObject.replaceAll("globalNav.js", "");
|
||||
forObject = forObject.replaceAll("TSB.js", "");
|
||||
forObject = forObject.replaceAll("https://images.dmzj.com/", "/manhua/images/");
|
||||
forObject = forObject.replaceAll("https://static.dmzj.com/", "/manhua/statics/");
|
||||
forObject = forObject + "<script>var browser={\n" +
|
||||
" versions:function(){\n" +
|
||||
" var u = window.navigator.userAgent;\n" +
|
||||
" return {\n" +
|
||||
" trident: u.indexOf('Trident') > -1, //IE内核\n" +
|
||||
" presto: u.indexOf('Presto') > -1, //opera内核\n" +
|
||||
" webKit: u.indexOf('AppleWebKit') > -1, //苹果、谷歌内核\n" +
|
||||
" gecko: u.indexOf('Gecko') > -1 && u.indexOf('KHTML') == -1, //火狐内核\n" +
|
||||
" mobile: !!u.match(/AppleWebKit.*Mobile.*/)||!!u.match(/AppleWebKit/), //是否为移动终端\n" +
|
||||
" ios: !!u.match(/\\(i[^;]+;( U;)? CPU.+Mac OS X/), //ios终端\n" +
|
||||
" android: u.indexOf('Android') > -1 || u.indexOf('Linux') > -1, //android终端或者uc浏览器\n" +
|
||||
" iPhone: u.indexOf('iPhone') > -1 || u.indexOf('Mac') > -1, //是否为iPhone或者安卓QQ浏览器\n" +
|
||||
" iPad: u.indexOf('iPad') > -1, //是否为iPad\n" +
|
||||
" webApp: u.indexOf('Safari') == -1 ,//是否为web应用程序，没有头部与底部\n" +
|
||||
" weixin: u.indexOf('MicroMessenger') == -1 //是否为微信浏览器\n" +
|
||||
" };\n" +
|
||||
" }()\n" +
|
||||
"};" +
|
||||
"</script>";
|
||||
forObject = forObject + "<script>$(function(){$(\"#app_manhua\").remove();" +
|
||||
"$('.btmBtnBox').remove();$('.red_box').remove()" +
|
||||
";$('.mainNav').remove();$('.wrap_last_head').remove();$('.wrap_last_mid').remove()" +
|
||||
";$('.comic_gd').remove();$('.comic_last').remove();$('.side_bar').remove()" +
|
||||
";$('.point_wrap').remove();$('.show').remove();$('.light').remove()" +
|
||||
";$('#sidePublic').remove();$('.side_public').remove()" +
|
||||
";$('.foot-detail').remove();$('#float_nav_type').remove()})</script>";
|
||||
// forObject = forObject.replaceAll("/manhua/", "https://www.dmzj.com/")
|
||||
// .replaceAll("笔趣岛", "酸味书屋")
|
||||
// .replaceAll("笔趣阁", "酸味书屋")
|
||||
// .replaceAll("class=\"mainNav independNav\"", "style='dispaly:none' class=\"mainNav independNav\"")
|
||||
// .replaceAll("</head>", "<script language=\"javascript\" type=\"text/javascript\" src=\"http://www.zinglizingli.xyz/js/wap_collect.js\"></script></head>")
|
||||
// .replaceFirst("</head>", "<script>" +
|
||||
// "var _hmt = _hmt || [];" +
|
||||
// "(function() {" +
|
||||
// " var hm = document.createElement(\"script\");" +
|
||||
// " hm.src = \"https://hm.baidu.com/hm.js?0bd7345ca6b694ea3dfbe87da008082e\";" +
|
||||
// " var s = document.getElementsByTagName(\"script\")[0]; " +
|
||||
// " s.parentNode.insertBefore(hm, s);" +
|
||||
// "})();" +
|
||||
// "</script></head>")
|
||||
// .replaceAll("<input type=\"image\" src=\"https://m.baidu.com/se/transcode/static/img/bgn.png\".*>", "")
|
||||
// .replaceAll("https://zhannei.baidu.com/cse", "http://m.zinglizingli.xyz")
|
||||
// .replaceAll("<a href=\"/.*/\">返回</a>", "<a href=\"javascript:history.go(-1)\">返回</a>")
|
||||
// .replaceAll("<a href=\".*\".*>加入书架</a>", "<a href=\"javascript:AddToFavorites(true);\">加入收藏</a>")
|
||||
// .replaceFirst("</head>", "<script>\n" +
|
||||
// "(function(){\n" +
|
||||
// " var bp = document.createElement('script');\n" +
|
||||
// " var curProtocol = window.location.protocol.split(':')[0];\n" +
|
||||
// " if (curProtocol === 'https') {\n" +
|
||||
// " bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';\n" +
|
||||
// " }\n" +
|
||||
// " else {\n" +
|
||||
// " bp.src = 'http://push.zhanzhang.baidu.com/push.js';\n" +
|
||||
// " }\n" +
|
||||
// " var s = document.getElementsByTagName(\"script\")[0];\n" +
|
||||
// " s.parentNode.insertBefore(bp, s);\n" +
|
||||
// "})();\n" +
|
||||
// "</script>\n</head>")//页面访问自动推送到百度
|
||||
// .replaceAll("<script.*wap\\.js.*script>", "")//去除广告
|
||||
;
|
||||
|
||||
|
||||
// if (requestURI.matches(SUANWEI_BOOK_HTML_REGEX)) {
|
||||
// Pattern pattern = Pattern.compile("<h1\\s+id=\"bqgmb_h1\">(.+)\\s+目录共\\d+章</h1>");
|
||||
// Matcher matcher = pattern.matcher(forObject);
|
||||
// String title = "";
|
||||
// if (matcher.find()) {
|
||||
// title = matcher.group(1);
|
||||
// }//<li class="sort"> 类别：武侠仙侠</li>
|
||||
// pattern = Pattern.compile("<p>作者：(.+)</p>");
|
||||
// matcher = pattern.matcher(forObject);
|
||||
// String author = "";
|
||||
// if (matcher.find()) {
|
||||
// author = matcher.group(1);
|
||||
// }
|
||||
// pattern = Pattern.compile("<a\\s+href=\"/list/\\d+_\\d+.*\">(.+)</a>");
|
||||
// matcher = pattern.matcher(forObject);
|
||||
// String sort = "";
|
||||
// if (matcher.find()) {
|
||||
// sort = matcher.group(1);
|
||||
// }
|
||||
// String desc = title + "，" + title + "小说最新章节免费在线阅读、最新章节列表，" + title + "小说最新更新免费提供，《" + title + "》是一本情节与文笔俱佳的" + sort + "小说，由作者" + author + "创建。";
|
||||
//
|
||||
// forObject = forObject.replaceFirst("<meta\\s+name=\"description\"\\s+content=\"[^>]+\"\\s*/?>", "");//[^>]+表示1个或多个不是>的字符
|
||||
// forObject = forObject.replaceFirst("<head>", "<head><meta name=\"description\" content=\"" + desc + "\"/>");
|
||||
//
|
||||
//
|
||||
// }
|
||||
//
|
||||
// if ("/".equals(requestURI)) {
|
||||
// forObject = forObject.replaceFirst("<meta\\s+name=\"description\"\\s+content=\"[^>]+\"\\s*/?>", "");//[^>]+表示1个或多个不是>的字符
|
||||
// forObject = forObject.replaceFirst("<head>", "<head><meta name=\"description\" content=\"酸味书屋致力于打造小说最全，更新最快的在线小说阅读网，本站收录了当前最火热的网络小说，提供无广告、高质量内容的小说服务，是广大网友最喜欢的温馨小说站。\">");
|
||||
//
|
||||
//
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
|
||||
|
||||
Map<String, String[]> oldParameterMap = req.getParameterMap();
|
||||
Map<String, String> newParameterMap = new HashMap<>();
|
||||
Set<Map.Entry<String, String[]>> entries = oldParameterMap.entrySet();
|
||||
for (Map.Entry<String, String[]> entry : entries) {
|
||||
newParameterMap.put(entry.getKey(), entry.getValue()[0]);
|
||||
}
|
||||
|
||||
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
|
||||
map.setAll(newParameterMap);
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
|
||||
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
|
||||
forObject = restTemplate.postForEntity("https://www.dmzj.com/" + requestURI.substring(8), request, String.class).getBody();
|
||||
// forObject = new String(forObject.getBytes("ISO-8859-1"),"utf-8");
|
||||
forObject = forObject.replaceAll("/manhua/", "https://www.dmzj.com/")
|
||||
.replaceAll("笔趣岛", "酸味书屋")
|
||||
.replaceAll("笔趣阁", "酸味书屋")
|
||||
.replaceFirst("</head>", "<script>\n" +
|
||||
"var _hmt = _hmt || [];" +
|
||||
"(function() {" +
|
||||
" var hm = document.createElement(\"script\");" +
|
||||
" hm.src = \"https://hm.baidu.com/hm.js?0bd7345ca6b694ea3dfbe87da008082e\";" +
|
||||
" var s = document.getElementsByTagName(\"script\")[0]; " +
|
||||
" s.parentNode.insertBefore(hm, s);" +
|
||||
"})();" +
|
||||
"</script></head>")
|
||||
.replaceAll("<a\\s+href=\"/bookcase.php\">书架</a>", "<a href=\"" + noteURI.get(new Random().nextInt(noteURI.size())) + "\">笔记</a>")
|
||||
.replaceAll("<a href=\"/.*/\">返回</a>", "<a href=\"javascript:history.go(-1)\">返回</a>")
|
||||
;
|
||||
forObject = setBookURIToHTML(forObject, SUANWEI_BOOK_REGEX);
|
||||
//resp.setCharacterEncoding("utf-8");
|
||||
//setContentType(postFix, resp);
|
||||
|
||||
}
|
||||
|
||||
|
||||
} catch (RuntimeException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
if (e instanceof HttpClientErrorException && (((HttpClientErrorException) e).getStatusCode() == HttpStatus.NOT_FOUND)) {
|
||||
//404
|
||||
resp.sendRedirect("/");
|
||||
return;
|
||||
} else {
|
||||
req.getRequestDispatcher("/mang.html").forward(servletRequest, servletResponse);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
//resp.setCharacterEncoding("utf-8");
|
||||
|
||||
}
|
||||
|
||||
resp.getWriter().print(forObject);
|
||||
return;
|
||||
}
|
||||
|
||||
private String addAttacDivForSearch(String forObject, String requestURI) {
|
||||
try {
|
||||
if (requestURI.endsWith(".html") || requestURI.equals("/")) {
|
||||
String hotNewsDiv = cacheUtil.get("hotNewsDiv");
|
||||
if (hotNewsDiv == null) {
|
||||
MultiValueMap<String, String> mmap = new LinkedMultiValueMap<>();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add("Host", "channel.chinanews.com");
|
||||
headers.add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36");
|
||||
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(mmap, headers);
|
||||
String body = restTemplate.postForEntity("http://channel.chinanews.com/cns/cjs/sh.shtml", request, String.class).getBody();
|
||||
Pattern pattern = Pattern.compile("specialcnsdata\\s*=\\s*\\{\"docs\":(.+)};\\s+newslist\\s*=\\s*specialcnsdata;");
|
||||
Matcher matcher = pattern.matcher(body);
|
||||
if (matcher.find()) {
|
||||
String jsonResult = matcher.group(1);
|
||||
if (jsonResult.length() > 5) {
|
||||
List<Map<String, String>> list = new ObjectMapper().readValue(jsonResult, List.class);
|
||||
StringBuilder hotContent = new StringBuilder();
|
||||
for (Map<String, String> map : list) {
|
||||
hotContent.append("<ul>\n");
|
||||
hotContent.append("<li>\n");
|
||||
hotContent.append(map.get("pubtime"));
|
||||
hotContent.append("</li>\n");
|
||||
hotContent.append("<li>\n");
|
||||
hotContent.append(map.get("title"));
|
||||
hotContent.append("</li>\n");
|
||||
hotContent.append("<li>\n");
|
||||
hotContent.append(map.get("content"));
|
||||
hotContent.append("</li>\n");
|
||||
hotContent.append("<li>\n");
|
||||
hotContent.append("<img src=\"" + map.get("galleryphoto") + "\"/>");
|
||||
hotContent.append("</li>\n");
|
||||
hotContent.append("</ul>\n");
|
||||
}
|
||||
hotNewsDiv = "<div style=\"position:fixed;top:0px;left:0px;z-index:-100;opacity:0\">" + hotContent.toString() + "</div>";
|
||||
cacheUtil.set("hotNewsDiv", hotNewsDiv, 60 * 60 * 24);
|
||||
forObject = forObject.replaceFirst("</body>", hotNewsDiv + "</body>");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
forObject = forObject.replaceFirst("</body>", hotNewsDiv + "</body>");
|
||||
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
} finally {
|
||||
|
||||
}
|
||||
|
||||
return forObject;
|
||||
}
|
||||
|
||||
private String setBookURIToHTML(String forObject, String regex) {
|
||||
String result = forObject;
|
||||
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
Matcher matcher = pattern.matcher(forObject);
|
||||
boolean isFind = matcher.find();
|
||||
if (isFind) {
|
||||
|
||||
while (isFind) {
|
||||
String booURI = matcher.group(1);
|
||||
String htmlBooURI = booURI.substring(0, booURI.length()) + ".html";
|
||||
result = result.replaceFirst(booURI + "/", htmlBooURI);
|
||||
isFind = matcher.find();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private String postBiquta(HttpServletRequest req, String realUrl, Map<String, String> otherParam) {
|
||||
String forObject;
|
||||
Map<String, String[]> oldParameterMap = req.getParameterMap();
|
||||
Map<String, String> newParameterMap = new HashMap<>();
|
||||
Set<Map.Entry<String, String[]>> entries = oldParameterMap.entrySet();
|
||||
for (Map.Entry<String, String[]> entry : entries) {
|
||||
newParameterMap.put(entry.getKey(), entry.getValue()[0]);
|
||||
}
|
||||
if (otherParam != null) {
|
||||
newParameterMap.putAll(otherParam);
|
||||
}
|
||||
|
||||
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
|
||||
map.setAll(newParameterMap);
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
|
||||
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
|
||||
forObject = restTemplate.postForEntity(realUrl, request, String.class).getBody();
|
||||
forObject = forObject.replaceAll("https://m.biquta.com", "http://m.zinglizingli.xyz")
|
||||
.replaceAll("笔趣阁", "看小说吧")
|
||||
.replaceAll("笔趣塔", "看小说吧")
|
||||
.replaceFirst("<title>看小说吧手机版-看小说吧</title>", "<title>看小说吧</title>")
|
||||
.replaceFirst("content=\"看小说吧\"", "content=\"小说阅读,小说排行,好看小说排行,热门小说排行,小说阅读手机版\"")
|
||||
.replaceFirst("<head>", "<head><meta name=\"shenma-site-verification\" content=\"5548d4bd962d5cdd4cf6aeba92b991a8_1565878917\">")
|
||||
.replaceAll("<a href=\"/bookcase.php\".*>我的书架</a>", "<a href=\"https://www.zinglizingli.xyz/book/searchSoftBook.html\">轻小说</a><a href=\"https://www.zinglizingli.xyz\">精品小说</a>")
|
||||
.replaceFirst("<a href=\"http://m.zinglizingli.xyz/tempcase.html\">阅读记录</a>", "<a href=\"/HotBook.apk\">客户端下载</a>")
|
||||
.replaceFirst("</head>", "<script>\n" +
|
||||
"var _hmt = _hmt || [];\n" +
|
||||
"(function() {\n" +
|
||||
" var hm = document.createElement(\"script\");\n" +
|
||||
" hm.src = \"https://hm.baidu.com/hm.js?b3a84b2ec6cc52dd088d735565b49644\";\n" +
|
||||
" var s = document.getElementsByTagName(\"script\")[0]; \n" +
|
||||
" s.parentNode.insertBefore(hm, s);\n" +
|
||||
"})();\n" +
|
||||
"</script>\n</head>")
|
||||
;
|
||||
|
||||
forObject = setBookURIToHTML(forObject, XIYANGYANG_BOOK_REGEX);
|
||||
return forObject;
|
||||
}
|
||||
|
||||
private void setContentType(String fileFix, HttpServletResponse resp) {
|
||||
String contentType = "text/html";
|
||||
switch (fileFix) {
|
||||
case "js": {
|
||||
contentType = "application/javascript";
|
||||
break;
|
||||
}
|
||||
case "css": {
|
||||
contentType = "text/css";
|
||||
break;
|
||||
}
|
||||
case "html": {
|
||||
contentType = "text/html";
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
resp.setContentType(contentType);
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,8 +1,8 @@
|
||||
package xyz.zinglizingli.common.schedule;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.Charsets;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@ -16,7 +16,6 @@ import xyz.zinglizingli.books.service.BookService;
|
||||
import xyz.zinglizingli.books.util.ExcutorUtils;
|
||||
import xyz.zinglizingli.common.utils.RestTemplateUtil;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
@ -24,17 +23,21 @@ import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static java.util.regex.Pattern.compile;
|
||||
|
||||
/**
|
||||
* 更新书籍章节内容定时任务
|
||||
*
|
||||
* @author 11797
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CrawlBooksSchedule {
|
||||
|
||||
private final BookService bookService;
|
||||
|
||||
private Logger log = LoggerFactory.getLogger(CrawlBooksSchedule.class);
|
||||
|
||||
|
||||
@Autowired
|
||||
private BookService bookService;
|
||||
|
||||
RestTemplate utf8RestTemplate = RestTemplateUtil.getInstance("utf-8");
|
||||
private RestTemplate utf8RestTemplate = RestTemplateUtil.getInstance(Charsets.UTF_8);
|
||||
|
||||
|
||||
@Value("${books.lowestScore}")
|
||||
@ -51,204 +54,187 @@ public class CrawlBooksSchedule {
|
||||
private String picSavePath;
|
||||
|
||||
|
||||
private boolean isExcuting = false;
|
||||
/**
|
||||
* 10分钟抓取一次
|
||||
*/
|
||||
@Scheduled(fixedRate = 1000 * 60 * 10)
|
||||
public void crawBooks() {
|
||||
|
||||
log.debug("crawlBooksSchedule执行中。。。。。。。。。。。。");
|
||||
|
||||
@Scheduled(fixedRate = 1000 * 60 * 60 * 3)
|
||||
public void crawBquge11BooksAtDay() throws Exception {
|
||||
if (!isExcuting) {
|
||||
isExcuting = true;
|
||||
log.debug("crawlBooksSchedule执行中。。。。。。。。。。。。");
|
||||
|
||||
while (true) {
|
||||
|
||||
try {
|
||||
switch (websiteType) {
|
||||
case 1: {
|
||||
updateBiqudaoBooks(0);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
updateBiquTaBooks(0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
Thread.sleep(1000 * 60 * 5);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
switch (websiteType) {
|
||||
case 1: {
|
||||
updateBiqudaoBooks(0);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
updateBiquTaBooks(0);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
private void updateBiquTaBooks(int finalI) {
|
||||
/**
|
||||
* 从笔趣塔更新
|
||||
*/
|
||||
private void updateBiquTaBooks(int bookClass) {
|
||||
String baseUrl = "https://m.biquta.la";
|
||||
String catBookListUrlBase = baseUrl + "/class/";
|
||||
|
||||
int page = 1;//起始页码
|
||||
int totalPage = page;
|
||||
String catBookListUrl = catBookListUrlBase + finalI + "/" + page + ".html";
|
||||
String forObject = getByHttpClient(catBookListUrl);
|
||||
String catBookListUrl = catBookListUrlBase + bookClass + "/" + 1 + ".html";
|
||||
String forObject = getByRestTemplate(catBookListUrl);
|
||||
if (forObject != null) {
|
||||
//匹配分页数<input type="text" class="page_txt" value="1/3019" size="5" name="txtPage" id="txtPage" />
|
||||
Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\"");
|
||||
Pattern pattern = compile("value=\"(\\d+)/(\\d+)\"");
|
||||
Matcher matcher = pattern.matcher(forObject);
|
||||
boolean isFind = matcher.find();
|
||||
System.out.println("匹配分页数" + isFind);
|
||||
if (isFind) {
|
||||
int currentPage = Integer.parseInt(matcher.group(1));
|
||||
totalPage = Integer.parseInt(matcher.group(2));
|
||||
//解析第一页书籍的数据
|
||||
Pattern bookPatten = Pattern.compile("href=\"/(\\d+_\\d+)/\"");
|
||||
parseBiquTaBook(bookPatten, forObject, finalI, baseUrl, true);
|
||||
Pattern bookPatten = compile("href=\"/(\\d+_\\d+)/\"");
|
||||
parseBiquTaBook(bookPatten, forObject, bookClass, baseUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void parseBiquTaBook(Pattern bookPatten, String forObject, int catNum, String baseUrl, boolean isUpdate) {
|
||||
Matcher matcher2 = bookPatten.matcher(forObject);
|
||||
boolean isFind = matcher2.find();
|
||||
Pattern scorePatten = Pattern.compile("<div\\s+class=\"score\">(\\d+\\.\\d+)分</div>");
|
||||
/**
|
||||
* 解析笔趣塔数据
|
||||
*/
|
||||
private void parseBiquTaBook(Pattern bookPatten, String forObject, int catNum, String baseUrl) {
|
||||
Matcher bookMatcher = bookPatten.matcher(forObject);
|
||||
|
||||
boolean isFind = bookMatcher.find();
|
||||
Pattern scorePatten = compile("<div\\s+class=\"score\">(\\d+\\.\\d+)分</div>");
|
||||
Matcher scoreMatch = scorePatten.matcher(forObject);
|
||||
boolean scoreFind = scoreMatch.find();
|
||||
|
||||
Pattern bookNamePatten = Pattern.compile("<p class=\"title\">([^/]+)</p>");
|
||||
Pattern bookNamePatten = compile("<p class=\"title\">([^/]+)</p>");
|
||||
Matcher bookNameMatch = bookNamePatten.matcher(forObject);
|
||||
boolean isBookNameMatch = bookNameMatch.find();
|
||||
|
||||
|
||||
System.out.println("匹配书籍url" + isFind);
|
||||
|
||||
System.out.println("匹配分数" + scoreFind);
|
||||
|
||||
while (isFind && scoreFind && isBookNameMatch) {
|
||||
|
||||
try {
|
||||
Float score = Float.parseFloat(scoreMatch.group(1));
|
||||
|
||||
if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说
|
||||
if (score < lowestScore) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String bokNum = matcher2.group(1);
|
||||
String bokNum = bookMatcher.group(1);
|
||||
String bookUrl = baseUrl + "/" + bokNum + "/";
|
||||
|
||||
String body = getByHttpClient(bookUrl);
|
||||
String body = getByRestTemplate(bookUrl);
|
||||
if (body != null) {
|
||||
|
||||
String bookName = bookNameMatch.group(1);
|
||||
Pattern authorPatten = Pattern.compile(">作者：([^/]+)<");
|
||||
Pattern authorPatten = compile(">作者：([^/]+)<");
|
||||
Matcher authoreMatch = authorPatten.matcher(body);
|
||||
if (authoreMatch.find()) {
|
||||
String author = authoreMatch.group(1);
|
||||
|
||||
Pattern statusPatten = Pattern.compile("状态：([^/]+)</li>");
|
||||
Pattern statusPatten = compile("状态：([^/]+)</li>");
|
||||
Matcher statusMatch = statusPatten.matcher(body);
|
||||
if (statusMatch.find()) {
|
||||
String status = statusMatch.group(1);
|
||||
|
||||
Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)</a>");
|
||||
Matcher updateTimeMatch = updateTimePatten.matcher(body);
|
||||
if (updateTimeMatch.find()) {
|
||||
String updateTimeStr = updateTimeMatch.group(1);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
|
||||
Date updateTime = format.parse(updateTimeStr);
|
||||
Pattern picPatten = Pattern.compile("<img src=\"([^>]+)\"\\s+onerror=\"this.src=");
|
||||
Matcher picMather = picPatten.matcher(body);
|
||||
if (picMather.find()) {
|
||||
String picSrc = picMather.group(1);
|
||||
Pattern updateTimePatten = compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)</a>");
|
||||
Matcher updateTimeMatch = updateTimePatten.matcher(body);
|
||||
if (updateTimeMatch.find()) {
|
||||
String updateTimeStr = updateTimeMatch.group(1);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
|
||||
Date updateTime = format.parse(updateTimeStr);
|
||||
Pattern picPatten = compile("<img src=\"([^>]+)\"\\s+onerror=\"this.src=");
|
||||
Matcher picMather = picPatten.matcher(body);
|
||||
if (picMather.find()) {
|
||||
String picSrc = picMather.group(1);
|
||||
|
||||
|
||||
|
||||
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
|
||||
Matcher descMatch = descPatten.matcher(body);
|
||||
if (descMatch.find()) {
|
||||
String desc = descMatch.group(1);
|
||||
Pattern descPatten = compile("class=\"review\">([^<]+)</p>");
|
||||
Matcher descMatch = descPatten.matcher(body);
|
||||
if (descMatch.find()) {
|
||||
String desc = descMatch.group(1);
|
||||
|
||||
|
||||
Book book = new Book();
|
||||
book.setAuthor(author);
|
||||
book.setCatid(catNum);
|
||||
book.setBookDesc(desc);
|
||||
book.setBookName(bookName);
|
||||
book.setScore(score > 10 ? 8.0f : score);
|
||||
book.setPicUrl(picSrc);
|
||||
book.setBookStatus(status);
|
||||
book.setUpdateTime(updateTime);
|
||||
Book book = new Book();
|
||||
book.setAuthor(author);
|
||||
book.setCatid(catNum);
|
||||
book.setBookDesc(desc);
|
||||
book.setBookName(bookName);
|
||||
book.setScore(score > 10 ? 8.0f : score);
|
||||
book.setPicUrl(picSrc);
|
||||
book.setBookStatus(status);
|
||||
book.setUpdateTime(updateTime);
|
||||
|
||||
List<BookIndex> indexList = new ArrayList<>();
|
||||
List<BookContent> contentList = new ArrayList<>();
|
||||
List<BookIndex> indexList = new ArrayList<>();
|
||||
List<BookContent> contentList = new ArrayList<>();
|
||||
|
||||
//读取目录
|
||||
Pattern indexPatten = Pattern.compile("<a\\s+href=\"(/du/\\d+_\\d+/)\">查看完整目录</a>");
|
||||
Matcher indexMatch = indexPatten.matcher(body);
|
||||
if (indexMatch.find()) {
|
||||
String indexUrl = baseUrl + indexMatch.group(1);
|
||||
String body2 = getByHttpClient(indexUrl);
|
||||
if (body2 != null) {
|
||||
Pattern indexListPatten = Pattern.compile("<a\\s+style=\"\"\\s+href=\"(/\\d+_\\d+/\\d+\\.html)\">([^/]+)</a>");
|
||||
Matcher indexListMatch = indexListPatten.matcher(body2);
|
||||
//读取目录
|
||||
Pattern indexPatten = compile("<a\\s+href=\"(/du/\\d+_\\d+/)\">查看完整目录</a>");
|
||||
Matcher indexMatch = indexPatten.matcher(body);
|
||||
if (indexMatch.find()) {
|
||||
String indexUrl = baseUrl + indexMatch.group(1);
|
||||
String body2 = getByRestTemplate(indexUrl);
|
||||
if (body2 != null) {
|
||||
Pattern indexListPatten = compile("<a\\s+style=\"\"\\s+href=\"(/\\d+_\\d+/\\d+\\.html)\">([^/]+)</a>");
|
||||
Matcher indexListMatch = indexListPatten.matcher(body2);
|
||||
|
||||
boolean isFindIndex = indexListMatch.find();
|
||||
boolean isFindIndex = indexListMatch.find();
|
||||
|
||||
int indexNum = 0;
|
||||
int indexNum = 0;
|
||||
|
||||
//查询该书籍已存在目录号
|
||||
List<Integer> hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author);
|
||||
//更新和插入分别开，插入只在凌晨做一次
|
||||
if ((isUpdate && hasIndexNum.size() > 0) || (!isUpdate && hasIndexNum.size() == 0)) {
|
||||
while (isFindIndex) {
|
||||
if (!hasIndexNum.contains(indexNum)) {
|
||||
//查询该书籍已存在目录号
|
||||
List<Integer> hasIndexNum = bookService.queryIndexCountByBookNameAndAuthor(bookName, author);
|
||||
//更新和插入分别开，插入只在凌晨做一次
|
||||
if (hasIndexNum.size() > 0) {
|
||||
while (isFindIndex) {
|
||||
if (!hasIndexNum.contains(indexNum)) {
|
||||
|
||||
String contentUrl = baseUrl + indexListMatch.group(1);
|
||||
String indexName = indexListMatch.group(2);
|
||||
String contentUrl = baseUrl + indexListMatch.group(1);
|
||||
String indexName = indexListMatch.group(2);
|
||||
|
||||
|
||||
//查询章节内容
|
||||
String body3 = getByHttpClient(contentUrl);
|
||||
if (body3 != null) {
|
||||
Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读");
|
||||
String start = "『章节错误,点此举报』";
|
||||
String end = "『加入书签，方便阅读』";
|
||||
String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end));
|
||||
//TODO插入章节目录和章节内容
|
||||
BookIndex bookIndex = new BookIndex();
|
||||
bookIndex.setIndexName(indexName);
|
||||
bookIndex.setIndexNum(indexNum);
|
||||
indexList.add(bookIndex);
|
||||
BookContent bookContent = new BookContent();
|
||||
bookContent.setContent(content);
|
||||
bookContent.setIndexNum(indexNum);
|
||||
contentList.add(bookContent);
|
||||
|
||||
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
//查询章节内容
|
||||
String body3 = getByRestTemplate(contentUrl);
|
||||
if (body3 != null) {
|
||||
String start = "『章节错误,点此举报』";
|
||||
String end = "『加入书签，方便阅读』";
|
||||
String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end));
|
||||
//TODO插入章节目录和章节内容
|
||||
BookIndex bookIndex = new BookIndex();
|
||||
bookIndex.setIndexName(indexName);
|
||||
bookIndex.setIndexNum(indexNum);
|
||||
indexList.add(bookIndex);
|
||||
BookContent bookContent = new BookContent();
|
||||
bookContent.setContent(content);
|
||||
bookContent.setIndexNum(indexNum);
|
||||
contentList.add(bookContent);
|
||||
|
||||
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
indexNum++;
|
||||
isFindIndex = indexListMatch.find();
|
||||
}
|
||||
|
||||
if (indexList.size() == contentList.size() && indexList.size() > 0) {
|
||||
ExcutorUtils.excuteFixedTask(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
bookService.saveBookAndIndexAndContent(book, indexList, contentList);
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
indexNum++;
|
||||
isFindIndex = indexListMatch.find();
|
||||
}
|
||||
|
||||
if (indexList.size() == contentList.size() && indexList.size() > 0) {
|
||||
ExcutorUtils.excuteFixedTask(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
bookService.saveBookAndIndexAndContent(book, indexList, contentList);
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -256,6 +242,9 @@ public class CrawlBooksSchedule {
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -270,8 +259,8 @@ public class CrawlBooksSchedule {
|
||||
e.printStackTrace();
|
||||
|
||||
} finally {
|
||||
matcher2.find();
|
||||
isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配
|
||||
bookMatcher.find();
|
||||
isFind = bookMatcher.find();
|
||||
scoreFind = scoreMatch.find();
|
||||
isBookNameMatch = bookNameMatch.find();
|
||||
}
|
||||
@ -280,27 +269,25 @@ public class CrawlBooksSchedule {
|
||||
}
|
||||
}
|
||||
|
||||
private void updateBiqudaoBooks(int finalI) {
|
||||
/**
|
||||
* 从笔趣岛更新
|
||||
*/
|
||||
private void updateBiqudaoBooks(int bookClass) {
|
||||
String baseUrl = "https://m.biqudao.com";
|
||||
String catBookListUrlBase = baseUrl + "/bqgeclass/";
|
||||
|
||||
int page = 1;//起始页码
|
||||
int totalPage = page;
|
||||
String catBookListUrl = catBookListUrlBase + finalI + "/" + page + ".html";
|
||||
String forObject = getByHttpClient(catBookListUrl);
|
||||
int page = 1;
|
||||
String catBookListUrl = catBookListUrlBase + bookClass + "/" + page + ".html";
|
||||
String forObject = getByRestTemplate(catBookListUrl);
|
||||
if (forObject != null) {
|
||||
//匹配分页数<input type="text" class="page_txt" value="1/3019" size="5" name="txtPage" id="txtPage" />
|
||||
Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\"");
|
||||
Pattern pattern = compile("value=\"(\\d+)/(\\d+)\"");
|
||||
Matcher matcher = pattern.matcher(forObject);
|
||||
boolean isFind = matcher.find();
|
||||
System.out.println("匹配分页数" + isFind);
|
||||
if (isFind) {
|
||||
int currentPage = Integer.parseInt(matcher.group(1));
|
||||
totalPage = Integer.parseInt(matcher.group(2));
|
||||
//解析第一页书籍的数据
|
||||
Pattern bookPatten = Pattern.compile("href=\"/(bqge\\d+)/\"");
|
||||
//白天更新
|
||||
parseBiquDaoBook(bookPatten, forObject, finalI, baseUrl, true);
|
||||
Pattern bookPatten = compile("href=\"/(bqge\\d+)/\"");
|
||||
parseBiquDaoBook(bookPatten, forObject, baseUrl);
|
||||
}
|
||||
}
|
||||
|
||||
@ -308,160 +295,53 @@ public class CrawlBooksSchedule {
|
||||
}
|
||||
|
||||
|
||||
//@Scheduled(cron = "0 0 2 * * ?")磁盘空间不足，暂时不抓新书
|
||||
//暂定2小说，只爬分类前3本书，一共3*7=21本书，爬等以后书籍多了之后，会适当缩短更新间隔
|
||||
public void crawBquge11BooksAtNight() throws Exception {
|
||||
final String baseUrl = "https://m.biqudao.com";
|
||||
log.debug("crawlBooksSchedule执行中。。。。。。。。。。。。");
|
||||
/**
|
||||
* 解析笔趣岛数据
|
||||
*/
|
||||
private void parseBiquDaoBook(Pattern bookPatten, String forObject, String baseUrl) {
|
||||
|
||||
|
||||
//①爬分类列表的书籍url和总页数
|
||||
// https:
|
||||
////m.biquta.la/class/1/1.html
|
||||
// https:
|
||||
////m.biquta.la/class/2/1.html
|
||||
// https:
|
||||
////m.biquta.la/class/2/2.html
|
||||
//
|
||||
//
|
||||
// https:
|
||||
////m.biquta.la/class/2/2.html
|
||||
//<input type = "text" class="page_txt" value = "2/1244" size = "5" name = "txtPage" id = "txtPage" >
|
||||
//
|
||||
//
|
||||
//<div class="bookinfo" >
|
||||
// <a href = "/14_14988/" >
|
||||
// <div class="detail" >
|
||||
// <p class="title" > 苍穹九变 </p >
|
||||
// <p class="author" > 作者：风起闲云</p >
|
||||
// </div >
|
||||
// <div class="score" > 7.5 分</div >
|
||||
// </a >
|
||||
// </div >
|
||||
//
|
||||
//
|
||||
//第一周期全部书拉取完后，可进行第二周期，只拉取前面几页的数据，拉取时间间隔变小
|
||||
log.debug("crawlBooksSchedule循环执行中。。。。。。。。。。。。");
|
||||
//List<Integer> classIdList = new ArrayList<>(Arrays.asList(new Integer[]{1,2,3,4,5,6,7}));
|
||||
// for (int i = 1; i <= 7; i++) {
|
||||
|
||||
// log.debug("crawlBooksSchedule分类"+i+"执行中。。。。。。。。。。。。");
|
||||
|
||||
// int finalI = i;
|
||||
/* new Thread(
|
||||
() -> {*/
|
||||
|
||||
try {
|
||||
//先随机更新分类
|
||||
//Random random = new Random();
|
||||
//int finalI = classIdList.get(new Random().nextInt(classIdList.size()));
|
||||
//classIdList.remove(finalI);
|
||||
int finalI = 0;
|
||||
//拼接分类URL
|
||||
int page = 1;//起始页码
|
||||
int totalPage = page;
|
||||
String catBookListUrl = baseUrl + "/bqgeclass/" + finalI + "/" + page + ".html";
|
||||
String forObject = getByHttpClient(catBookListUrl);
|
||||
if (forObject != null) {
|
||||
//匹配分页数<input type="text" class="page_txt" value="1/3019" size="5" name="txtPage" id="txtPage" />
|
||||
Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\"");
|
||||
Matcher matcher = pattern.matcher(forObject);
|
||||
boolean isFind = matcher.find();
|
||||
System.out.println("匹配分页数" + isFind);
|
||||
if (isFind) {
|
||||
int currentPage = Integer.parseInt(matcher.group(1));
|
||||
totalPage = Integer.parseInt(matcher.group(2));
|
||||
//解析第一页书籍的数据
|
||||
Pattern bookPatten = Pattern.compile("href=\"/(bqge\\d+)/\"");
|
||||
//晚上插入
|
||||
parseBiquDaoBook(bookPatten, forObject, finalI, baseUrl, false);
|
||||
while (currentPage < totalPage) {
|
||||
if (new Date().getHours() > 5) {
|
||||
break;
|
||||
}
|
||||
catBookListUrl = baseUrl + "/bqgeclass/" + finalI + "/" + (currentPage + 1) + ".html";
|
||||
forObject = getByHttpClient(catBookListUrl);
|
||||
if (forObject != null) {
|
||||
//匹配分页数
|
||||
matcher = pattern.matcher(forObject);
|
||||
isFind = matcher.find();
|
||||
|
||||
if (isFind) {
|
||||
currentPage = Integer.parseInt(matcher.group(1));
|
||||
totalPage = Integer.parseInt(matcher.group(2));
|
||||
parseBiquDaoBook(bookPatten, forObject, finalI, baseUrl, false);
|
||||
}
|
||||
} else {
|
||||
currentPage++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
/* }
|
||||
).start();*/
|
||||
|
||||
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
|
||||
private void parseBiquDaoBook(Pattern bookPatten, String forObject, int catNum, String baseUrl, boolean isUpdate) {
|
||||
|
||||
Matcher matcher2 = bookPatten.matcher(forObject);
|
||||
boolean isFind = matcher2.find();
|
||||
Pattern scorePatten = Pattern.compile("<div\\s+class=\"score\">(\\d+\\.\\d+)分</div>");
|
||||
Matcher bookMatcher = bookPatten.matcher(forObject);
|
||||
boolean isFind = bookMatcher.find();
|
||||
Pattern scorePatten = compile("<div\\s+class=\"score\">(\\d+\\.\\d+)分</div>");
|
||||
Matcher scoreMatch = scorePatten.matcher(forObject);
|
||||
boolean scoreFind = scoreMatch.find();
|
||||
|
||||
Pattern bookNamePatten = Pattern.compile("<p class=\"title\">([^/]+)</p>");
|
||||
Pattern bookNamePatten = compile("<p class=\"title\">([^/]+)</p>");
|
||||
Matcher bookNameMatch = bookNamePatten.matcher(forObject);
|
||||
boolean isBookNameMatch = bookNameMatch.find();
|
||||
|
||||
|
||||
System.out.println("匹配书籍url" + isFind);
|
||||
|
||||
System.out.println("匹配分数" + scoreFind);
|
||||
|
||||
while (isFind && scoreFind && isBookNameMatch) {
|
||||
|
||||
try {
|
||||
Float score = Float.parseFloat(scoreMatch.group(1));
|
||||
|
||||
if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说
|
||||
if (score < lowestScore) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String bokNum = matcher2.group(1);
|
||||
String bokNum = bookMatcher.group(1);
|
||||
String bookUrl = baseUrl + "/" + bokNum + "/";
|
||||
|
||||
String body = getByHttpClient(bookUrl);
|
||||
String body = getByRestTemplate(bookUrl);
|
||||
if (body != null) {
|
||||
|
||||
String bookName = bookNameMatch.group(1);
|
||||
Pattern authorPatten = Pattern.compile("<li class=\"author\">作者：([^/]+)</li>");
|
||||
Pattern authorPatten = compile("<li class=\"author\">作者：([^/]+)</li>");
|
||||
Matcher authoreMatch = authorPatten.matcher(body);
|
||||
if (authoreMatch.find()) {
|
||||
String author = authoreMatch.group(1);
|
||||
|
||||
Pattern statusPatten = Pattern.compile("状态：([^/]+)</li>");
|
||||
Pattern statusPatten = compile("状态：([^/]+)</li>");
|
||||
Matcher statusMatch = statusPatten.matcher(body);
|
||||
if (statusMatch.find()) {
|
||||
String status = statusMatch.group(1);
|
||||
|
||||
Pattern catPatten = Pattern.compile("类别：([^/]+)</li>");
|
||||
Pattern catPatten = compile("类别：([^/]+)</li>");
|
||||
Matcher catMatch = catPatten.matcher(body);
|
||||
if (catMatch.find()) {
|
||||
String catName = catMatch.group(1);
|
||||
int catNum;
|
||||
switch (catName) {
|
||||
case "玄幻奇幻": {
|
||||
catNum = 1;
|
||||
break;
|
||||
}
|
||||
case "武侠仙侠": {
|
||||
catNum = 2;
|
||||
break;
|
||||
@ -491,19 +371,19 @@ public class CrawlBooksSchedule {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)</a>");
|
||||
Pattern updateTimePatten = compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)</a>");
|
||||
Matcher updateTimeMatch = updateTimePatten.matcher(body);
|
||||
if (updateTimeMatch.find()) {
|
||||
String updateTimeStr = updateTimeMatch.group(1);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
|
||||
Date updateTime = format.parse(updateTimeStr);
|
||||
Pattern picPatten = Pattern.compile("<img src=\"([^>]+)\"\\s+onerror=\"this.src=");
|
||||
Pattern picPatten = compile("<img src=\"([^>]+)\"\\s+onerror=\"this.src=");
|
||||
Matcher picMather = picPatten.matcher(body);
|
||||
if (picMather.find()) {
|
||||
String picSrc = picMather.group(1);
|
||||
|
||||
|
||||
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
|
||||
Pattern descPatten = compile("class=\"review\">([^<]+)</p>");
|
||||
Matcher descMatch = descPatten.matcher(body);
|
||||
if (descMatch.find()) {
|
||||
String desc = descMatch.group(1);
|
||||
@ -523,13 +403,13 @@ public class CrawlBooksSchedule {
|
||||
List<BookContent> contentList = new ArrayList<>();
|
||||
|
||||
//读取目录
|
||||
Pattern indexPatten = Pattern.compile("<a\\s+href=\"(/bqge\\d+/all\\.html)\">查看完整目录</a>");
|
||||
Pattern indexPatten = compile("<a\\s+href=\"(/bqge\\d+/all\\.html)\">查看完整目录</a>");
|
||||
Matcher indexMatch = indexPatten.matcher(body);
|
||||
if (indexMatch.find()) {
|
||||
String indexUrl = baseUrl + indexMatch.group(1);
|
||||
String body2 = getByHttpClient(indexUrl);
|
||||
String body2 = getByRestTemplate(indexUrl);
|
||||
if (body2 != null) {
|
||||
Pattern indexListPatten = Pattern.compile("<a[^/]+style[^/]+href=\"(/bqge\\d+/\\d+\\.html)\">([^/]+)</a>");
|
||||
Pattern indexListPatten = compile("<a[^/]+style[^/]+href=\"(/bqge\\d+/\\d+\\.html)\">([^/]+)</a>");
|
||||
Matcher indexListMatch = indexListPatten.matcher(body2);
|
||||
|
||||
boolean isFindIndex = indexListMatch.find();
|
||||
@ -537,9 +417,9 @@ public class CrawlBooksSchedule {
|
||||
int indexNum = 0;
|
||||
|
||||
//查询该书籍已存在目录号
|
||||
List<Integer> hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author);
|
||||
List<Integer> hasIndexNum = bookService.queryIndexCountByBookNameAndAuthor(bookName, author);
|
||||
//更新和插入分别开，插入只在凌晨做一次
|
||||
if ((isUpdate && hasIndexNum.size() > 0) || (!isUpdate && hasIndexNum.size() == 0)) {
|
||||
if (hasIndexNum.size() > 0) {
|
||||
while (isFindIndex) {
|
||||
if (!hasIndexNum.contains(indexNum)) {
|
||||
|
||||
@ -548,9 +428,8 @@ public class CrawlBooksSchedule {
|
||||
|
||||
|
||||
//查询章节内容
|
||||
String body3 = getByHttpClient(contentUrl);
|
||||
String body3 = getByRestTemplate(contentUrl);
|
||||
if (body3 != null) {
|
||||
Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读");
|
||||
String start = "『章节错误,点此举报』";
|
||||
String end = "『加入书签，方便阅读』";
|
||||
String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end));
|
||||
@ -576,12 +455,7 @@ public class CrawlBooksSchedule {
|
||||
}
|
||||
|
||||
if (indexList.size() == contentList.size() && indexList.size() > 0) {
|
||||
ExcutorUtils.excuteFixedTask(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
bookService.saveBookAndIndexAndContent(book, indexList, contentList);
|
||||
}
|
||||
});
|
||||
ExcutorUtils.excuteFixedTask(() -> bookService.saveBookAndIndexAndContent(book, indexList, contentList));
|
||||
|
||||
}
|
||||
}
|
||||
@ -610,8 +484,8 @@ public class CrawlBooksSchedule {
|
||||
e.printStackTrace();
|
||||
|
||||
} finally {
|
||||
matcher2.find();
|
||||
isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配
|
||||
bookMatcher.find();
|
||||
isFind = bookMatcher.find();
|
||||
scoreFind = scoreMatch.find();
|
||||
isBookNameMatch = bookNameMatch.find();
|
||||
}
|
||||
@ -621,19 +495,8 @@ public class CrawlBooksSchedule {
|
||||
|
||||
}
|
||||
|
||||
private String getByHttpClient(String url) {
|
||||
private String getByRestTemplate(String url) {
|
||||
try {
|
||||
/* HttpClient httpClient = new DefaultHttpClient();
|
||||
HttpGet getReq = new HttpGet(catBookListUrl);
|
||||
getReq.setHeader("user-agent", "Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1");
|
||||
HttpResponse execute = httpClient.execute(getReq);
|
||||
if (execute.getStatusLine().getStatusCode() == HttpStatus.OK.value()) {
|
||||
HttpEntity entity = execute.getEntity();
|
||||
return EntityUtils.toString(entity, "utf-8");
|
||||
} else {
|
||||
return null;
|
||||
}*/
|
||||
//经测试restTemplate比httpClient效率高出很多倍，所有选择restTemplate
|
||||
ResponseEntity<String> forEntity = utf8RestTemplate.getForEntity(url, String.class);
|
||||
if (forEntity.getStatusCode() == HttpStatus.OK) {
|
||||
return forEntity.getBody();
|
||||
@ -641,173 +504,10 @@ public class CrawlBooksSchedule {
|
||||
return null;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
log.error(e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
* 解析书籍详情之后的页面
|
||||
*/
|
||||
private void parseBook(Pattern bookPatten, String forObject, RestTemplate restTemplate, int catNum, String baseUrl) throws ParseException {
|
||||
Matcher matcher2 = bookPatten.matcher(forObject);
|
||||
boolean isFind = matcher2.find();
|
||||
Pattern scorePatten = Pattern.compile("<div\\s+class=\"score\">(\\d+\\.\\d+)分</div>");
|
||||
Matcher scoreMatch = scorePatten.matcher(forObject);
|
||||
boolean scoreFind = scoreMatch.find();
|
||||
|
||||
Pattern bookNamePatten = Pattern.compile("<p class=\"title\">([^/]+)</p>");
|
||||
Matcher bookNameMatch = bookNamePatten.matcher(forObject);
|
||||
boolean isBookNameMatch = bookNameMatch.find();
|
||||
|
||||
Pattern authorPatten = Pattern.compile(">作者：([^/]+)<");
|
||||
Matcher authoreMatch = authorPatten.matcher(forObject);
|
||||
boolean isFindAuthor = authoreMatch.find();
|
||||
|
||||
|
||||
System.out.println("匹配书籍url" + isFind);
|
||||
|
||||
System.out.println("匹配分数" + scoreFind);
|
||||
while (isFind && scoreFind && isBookNameMatch && isFindAuthor) {
|
||||
|
||||
try {
|
||||
Float score = Float.parseFloat(scoreMatch.group(1));
|
||||
|
||||
if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说
|
||||
continue;
|
||||
}
|
||||
String bookName = bookNameMatch.group(1);
|
||||
String author = authoreMatch.group(1);
|
||||
|
||||
String bokNum = matcher2.group(1);
|
||||
String bookUrl = baseUrl + "/" + bokNum + "/";
|
||||
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(bookUrl, String.class);
|
||||
if (forEntity.getStatusCode() == HttpStatus.OK) {
|
||||
|
||||
String body = forEntity.getBody();
|
||||
|
||||
Pattern statusPatten = Pattern.compile("状态：([^/]+)</li>");
|
||||
Matcher statusMatch = statusPatten.matcher(body);
|
||||
if (statusMatch.find()) {
|
||||
String status = statusMatch.group(1);
|
||||
Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)</a>");
|
||||
Matcher updateTimeMatch = updateTimePatten.matcher(body);
|
||||
if (updateTimeMatch.find()) {
|
||||
String updateTimeStr = updateTimeMatch.group(1);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
|
||||
Date updateTime = format.parse(updateTimeStr);
|
||||
Pattern picPatten = Pattern.compile("<img src=\"([^>]+)\"\\s+onerror=\"this.src=");
|
||||
Matcher picMather = picPatten.matcher(body);
|
||||
if (picMather.find()) {
|
||||
String picSrc = picMather.group(1);
|
||||
|
||||
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
|
||||
Matcher descMatch = descPatten.matcher(body);
|
||||
if (descMatch.find()) {
|
||||
String desc = descMatch.group(1);
|
||||
|
||||
|
||||
Book book = new Book();
|
||||
book.setAuthor(author);
|
||||
book.setCatid(catNum);
|
||||
book.setBookDesc(desc);
|
||||
book.setBookName(bookName);
|
||||
book.setScore(score > 10 ? 8.0f : score);
|
||||
book.setPicUrl(picSrc);
|
||||
book.setBookStatus(status);
|
||||
book.setUpdateTime(updateTime);
|
||||
|
||||
List<BookIndex> indexList = new ArrayList<>();
|
||||
List<BookContent> contentList = new ArrayList<>();
|
||||
|
||||
//读取目录
|
||||
Pattern indexPatten = Pattern.compile("<a\\s+href=\"(/du/\\d+_\\d+/)\">查看完整目录</a>");
|
||||
Matcher indexMatch = indexPatten.matcher(body);
|
||||
if (indexMatch.find()) {
|
||||
String indexUrl = baseUrl + indexMatch.group(1);
|
||||
ResponseEntity<String> forEntity1 = restTemplate.getForEntity(indexUrl, String.class);
|
||||
if (forEntity1.getStatusCode() == HttpStatus.OK) {
|
||||
String body2 = forEntity1.getBody();
|
||||
Pattern indexListPatten = Pattern.compile("<a\\s+style=\"\"\\s+href=\"(/\\d+_\\d+/\\d+\\.html)\">([^/]+)</a>");
|
||||
Matcher indexListMatch = indexListPatten.matcher(body2);
|
||||
|
||||
boolean isFindIndex = indexListMatch.find();
|
||||
|
||||
int indexNum = 0;
|
||||
|
||||
//查询该书籍已存在目录号
|
||||
List<Integer> hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author);
|
||||
|
||||
while (isFindIndex) {
|
||||
if (!hasIndexNum.contains(indexNum)) {
|
||||
|
||||
String contentUrl = baseUrl + indexListMatch.group(1);
|
||||
String indexName = indexListMatch.group(2);
|
||||
|
||||
|
||||
//查询章节内容
|
||||
ResponseEntity<String> forEntity2 = restTemplate.getForEntity(contentUrl, String.class);
|
||||
if (forEntity2.getStatusCode() == HttpStatus.OK) {
|
||||
String body3 = forEntity2.getBody();
|
||||
Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读");
|
||||
String start = "『章节错误,点此举报』";
|
||||
String end = "『加入书签，方便阅读』";
|
||||
String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end));
|
||||
//TODO插入章节目录和章节内容
|
||||
BookIndex bookIndex = new BookIndex();
|
||||
bookIndex.setIndexName(indexName);
|
||||
bookIndex.setIndexNum(indexNum);
|
||||
indexList.add(bookIndex);
|
||||
BookContent bookContent = new BookContent();
|
||||
bookContent.setContent(content);
|
||||
bookContent.setIndexNum(indexNum);
|
||||
contentList.add(bookContent);
|
||||
|
||||
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
indexNum++;
|
||||
isFindIndex = indexListMatch.find();
|
||||
}
|
||||
if (indexList.size() == contentList.size() && indexList.size() > 0) {
|
||||
bookService.saveBookAndIndexAndContent(book, indexList, contentList);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
|
||||
e.printStackTrace();
|
||||
|
||||
} finally {
|
||||
matcher2.find();
|
||||
isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配
|
||||
scoreFind = scoreMatch.find();
|
||||
isBookNameMatch = bookNameMatch.find();
|
||||
isFindAuthor = authoreMatch.find();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
package xyz.zinglizingli.common.schedule;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import xyz.zinglizingli.books.constant.CacheKeyConstans;
|
||||
import xyz.zinglizingli.books.service.MailService;
|
||||
import xyz.zinglizingli.books.util.RandomValueUtil;
|
||||
import xyz.zinglizingli.common.cache.CommonCacheUtil;
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
|
||||
/*
|
||||
主动推送：最为快速的提交方式，
|
||||
建议您将站点当天新产出链接立即通过此方式推送给百度，
|
||||
以保证新链接可以及时被百度收录。
|
||||
*/
|
||||
@Service
|
||||
public class SendEmaillSchedule {
|
||||
|
||||
@Autowired
|
||||
private CommonCacheUtil cacheUtil;
|
||||
|
||||
@Autowired
|
||||
private MailService mailService;
|
||||
|
||||
|
||||
private Logger log = LoggerFactory.getLogger(SendEmaillSchedule.class);
|
||||
|
||||
|
||||
// @Scheduled(fixedRate = 1000*60*60*24)
|
||||
public void sendEmaill() {
|
||||
System.out.println("SendEmaillSchedule。。。。。。。。。。。。。。。");
|
||||
|
||||
for(int i = 0 ; i < 1000; i++){
|
||||
String email = RandomValueUtil.getEmail();
|
||||
if(cacheUtil.get(CacheKeyConstans.EMAIL_URL_PREFIX_KEY+email)!=null){
|
||||
continue;
|
||||
}
|
||||
cacheUtil.setObject(CacheKeyConstans.EMAIL_URL_PREFIX_KEY+email,email,60*60*24*30);
|
||||
String subject = "推荐一个看小说的弹幕网站";
|
||||
String content = "精品小说楼是国内优秀的<b style='color:red'>小说弹幕网站</b>,精品小说楼提供海量热门网络小说,日本轻小说,国产轻小说,动漫小说,轻小说<b style='color:red'>在线阅读</b>和<b style='color:red'>TXT小说下载</b>,致力于网络精品小说的收集,智能计算小说评分,打造小说<b style='color:red'>精品排行榜</b>,致力于<b style='color:red'>无广告无弹窗</b>的小说阅读环境。" +
|
||||
"<br/><a href='https://www.zinglizingli.xyz/'>点击进入</a>"
|
||||
+"<br/><img src='https://www.zinglizingli.xyz/me/assets/images/work001-01.jpg'>";
|
||||
mailService.sendHtmlMail(email, subject, content);
|
||||
try {
|
||||
Thread.sleep(new Random().nextInt(1000*60*10)+1000*60);
|
||||
} catch (InterruptedException e) {
|
||||
log.error(e.getMessage(),e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
package xyz.zinglizingli.common.schedule;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import xyz.zinglizingli.books.service.BookService;
|
||||
import xyz.zinglizingli.common.cache.CommonCacheUtil;
|
||||
import xyz.zinglizingli.common.utils.RestTemplateUtil;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
|
||||
/*
|
||||
主动推送：最为快速的提交方式，
|
||||
建议您将站点当天新产出链接立即通过此方式推送给百度，
|
||||
以保证新链接可以及时被百度收录。
|
||||
*/
|
||||
@Service
|
||||
public class SendUrlSchedule {
|
||||
|
||||
@Autowired
|
||||
private CommonCacheUtil cacheUtil;
|
||||
|
||||
@Autowired
|
||||
private BookService bookService;
|
||||
|
||||
@Value("${baidu.record.ids}")
|
||||
private String recordedIds;
|
||||
|
||||
private Logger log = LoggerFactory.getLogger(SendUrlSchedule.class);
|
||||
|
||||
|
||||
//@Scheduled(cron = "0 0 1 * * 1")
|
||||
public void sendAllBookToBaidu() {
|
||||
System.out.println("sendAllBookToBaidu。。。。。。。。。。。。。。。");
|
||||
|
||||
List<String> recordedIdsList = Arrays.asList(recordedIds.split(","));
|
||||
List<String> idList = bookService.queryEndBookIdList();
|
||||
RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");
|
||||
|
||||
|
||||
String reqBody = "";
|
||||
for (String id : idList) {
|
||||
try {
|
||||
if (!recordedIdsList.contains(id)) {
|
||||
reqBody += ("https://www.zinglizingli.xyz/book/" + id + ".html" + "\n");
|
||||
//reqBody+=("http://www.zinglizingli.xyz/book/"+id+".html"+"\n");
|
||||
if (reqBody.length() > 2000) {
|
||||
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.TEXT_PLAIN);
|
||||
//headers.add("User-Agent","curl/7.12.1");
|
||||
headers.add("Host", "data.zz.baidu.com");
|
||||
headers.setContentLength(reqBody.length());
|
||||
HttpEntity<String> request = new HttpEntity<>(reqBody, headers);
|
||||
System.out.println("推送数据：" + reqBody);
|
||||
ResponseEntity<String> stringResponseEntity = restTemplate.postForEntity("http://data.zz.baidu.com/urls?site=www.zinglizingli.xyz&token=IuK7oVrPKe3U606x", request, String.class);
|
||||
System.out.println("推送URL结果：code:" + stringResponseEntity.getStatusCode().value() + ",body:" + stringResponseEntity.getBody());
|
||||
Thread.sleep(1000 * 10);
|
||||
System.out.println("推送数据：" + reqBody);
|
||||
stringResponseEntity = restTemplate.postForEntity("http://data.zz.baidu.com/urls?appid=1643715155923937&token=fkEcTlId6Cf21Sz3&type=batch", request, String.class);
|
||||
System.out.println("推送URL结果：code:" + stringResponseEntity.getStatusCode().value() + ",body:" + stringResponseEntity.getBody());
|
||||
|
||||
reqBody = "";
|
||||
Thread.sleep(1000 * 10);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,497 +0,0 @@
|
||||
package xyz.zinglizingli.common.schedule;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import xyz.zinglizingli.books.po.Book;
|
||||
import xyz.zinglizingli.books.service.BookService;
|
||||
import xyz.zinglizingli.common.cache.CommonCacheUtil;
|
||||
import xyz.zinglizingli.common.utils.RestTemplateUtil;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Service
|
||||
public class SendWeiboSchedule {
|
||||
|
||||
@Autowired
|
||||
private CommonCacheUtil cacheUtil;
|
||||
|
||||
@Autowired
|
||||
private BookService bookService;
|
||||
|
||||
private Logger log = LoggerFactory.getLogger(SendWeiboSchedule.class);
|
||||
|
||||
private boolean isExcuting = false;//是否正在执行
|
||||
|
||||
private long excuteNum = 0;
|
||||
|
||||
@Value("${search.schedule.isRunExcute}")
|
||||
private String isRunExcute;//是否在运行时就执行sendAtNight定时器
|
||||
|
||||
@Value("${browser.cookie}")
|
||||
private String cookieStr;
|
||||
|
||||
private static final String BOOKNAME_CACHE_PREFIX = "bookName_Cache_Prefix:";
|
||||
|
||||
|
||||
//@Scheduled(fixedRate = 1000 * 60 * 35)
|
||||
public void sendAtDay() {
|
||||
log.debug("sendWeoboSchedule执行中。。。。。。。。。。。。");
|
||||
if (!isExcuting) {
|
||||
isExcuting = true;
|
||||
excuteNum++;
|
||||
//long sleepMillis = 1000 * 60 * 5;
|
||||
long sleepMillis = 1000 * 60 * 5;
|
||||
try {
|
||||
|
||||
String name;
|
||||
String desc;
|
||||
String author;
|
||||
String bookNum;
|
||||
String resultCode;
|
||||
long realSleepMillis;
|
||||
|
||||
RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");
|
||||
|
||||
//发送数据库中的一篇文章
|
||||
Map<String, Object> dataMap = bookService.queryNewstBook();
|
||||
log.debug("dataMap大小：" + dataMap.size());
|
||||
if (dataMap.size() > 1) {
|
||||
Book book = (Book) dataMap.get("book");
|
||||
String newstIndexName = (String) dataMap.get("newstIndexName");
|
||||
name = newstIndexName + "_" + book.getBookName();
|
||||
desc = book.getBookDesc();
|
||||
author = book.getAuthor();
|
||||
bookNum = new Random().nextInt(100) + "";
|
||||
realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000;
|
||||
log.debug("发送微博书籍名：" + book.getBookName());
|
||||
if (!name.equals(cacheUtil.get(BOOKNAME_CACHE_PREFIX + name))) {
|
||||
resultCode = sendOneSiteWeibo(restTemplate, book.getBookName(), newstIndexName, author, desc, "精品小说楼", bookNum, "https://www.zinglizingli.xyz/book/" + book.getId() + ".html");
|
||||
log.debug("发送微博书籍名：" + book.getBookName() + " 状态码：" + resultCode);
|
||||
if ("{\"code\":\"A00006\"}".equals(resultCode)) {
|
||||
cacheUtil.set(BOOKNAME_CACHE_PREFIX + name, name, 60 * 60 * 24 * 30);
|
||||
}
|
||||
Thread.sleep(realSleepMillis + 32);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//分享喜羊羊小说网
|
||||
String url2 = "http://m.zinglizingli.xyz/class/0/1.html";
|
||||
|
||||
ResponseEntity<String> forEntity2 = restTemplate.getForEntity(url2, String.class);
|
||||
|
||||
|
||||
String forObject2 = forEntity2.getBody();
|
||||
Pattern pattern = Pattern.compile("<div class=\"bookinfo\">" +
|
||||
"\\s*<a href=\"/(\\d*_\\d*)\\.html\">" +
|
||||
"\\s*<div class=\"detail\">" +
|
||||
"\\s*<p class=\"title\">(.*)</p>" +
|
||||
"\\s*<p class=\"author\">(.*)</p>" +
|
||||
"\\s*</div>" +
|
||||
"\\s*<div class=\"score\">(\\d*\\.\\d*)分</div>" +
|
||||
"\\s*</a>" +
|
||||
"\\s*</div>" +
|
||||
"\\s*<p class=\"review\"><span class=\"longview\"></span>((.*))</p>\\s*" +
|
||||
"</div>");
|
||||
Matcher match = pattern.matcher(forObject2);
|
||||
boolean isFind = match.find();
|
||||
if (isFind) {
|
||||
while (isFind) {
|
||||
|
||||
float score = Float.parseFloat(match.group(4));
|
||||
|
||||
if (score >= 7.0) {
|
||||
|
||||
bookNum = match.group(1);
|
||||
String href = "http://m.zinglizingli.xyz/" + bookNum + ".html";
|
||||
name = match.group(2);
|
||||
if (!name.equals(cacheUtil.get(BOOKNAME_CACHE_PREFIX + name))) {
|
||||
author = match.group(3);
|
||||
desc = match.group(5);
|
||||
|
||||
resultCode = sendOneSiteWeibo(restTemplate, name, "", author, desc, "看小说吧", bookNum, href);
|
||||
if ("{\"code\":\"A00006\"}".equals(resultCode)) {
|
||||
cacheUtil.set(BOOKNAME_CACHE_PREFIX + name, name, 60 * 60 * 24 * 30);
|
||||
}
|
||||
|
||||
realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000;
|
||||
Thread.sleep(realSleepMillis);
|
||||
|
||||
}
|
||||
}
|
||||
isFind = match.find();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
|
||||
} finally {
|
||||
isExcuting = false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//@Scheduled(fixedRate = 1000 * 60 * 35)
|
||||
//@Scheduled(fixedRate = 1000 * 60 * 5)
|
||||
/*public void sendAtDay() {
|
||||
if (!isExcuting) {
|
||||
isExcuting = true;
|
||||
excuteNum++;
|
||||
//long sleepMillis = 1000 * 60 * 5;
|
||||
long sleepMillis = 1000 * 60 * 25;
|
||||
try {
|
||||
RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");
|
||||
//分享酸味书屋
|
||||
String url = "http://www.zinglizingli.xyz/paihangbang_lastupdate/1.html";
|
||||
|
||||
//分享喜羊羊小说网
|
||||
String url2 = "http://m.zinglizingli.xyz/class/0/1.html";
|
||||
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(url, String.class);
|
||||
ResponseEntity<String> forEntity2 = restTemplate.getForEntity(url2, String.class);
|
||||
|
||||
String forObject = forEntity.getBody();
|
||||
Pattern pattern = Pattern.compile("<li class=\"tjxs\">\n" +
|
||||
"<span class=\"xsm\"><a href=\"/(\\d*_\\d*)\\.html\">(.*)</a></span>\n" +
|
||||
"<span class=\"\">(.*)</span>\n" +
|
||||
"<span class=\"\">(.*)\\.\\.\\.</span>\n" +
|
||||
"<span class=\"tjrs\"><i>(\\d*)</i>人在看</span>\n" +
|
||||
"</li>");
|
||||
Matcher match = pattern.matcher(forObject);
|
||||
boolean isFind = match.find();
|
||||
if (isFind) {
|
||||
while (isFind) {
|
||||
|
||||
int lookNum = Integer.parseInt(match.group(5));
|
||||
if (lookNum > 5000) {
|
||||
String bookNum = match.group(1);
|
||||
String href = "http://www.zinglizingli.xyz/" + bookNum + ".html";
|
||||
String name = match.group(2);
|
||||
log.debug(excuteNum + ":" + name + "_BOOKNAME_CACHE:" + cacheUtil.get(BOOKNAME_CACHE_PREFIX + name));
|
||||
if (!name.equals(cacheUtil.get(BOOKNAME_CACHE_PREFIX + name))) {
|
||||
|
||||
String author = match.group(3);
|
||||
String desc = match.group(4);
|
||||
log.debug(excuteNum + ":" + name);
|
||||
String resultCode = sendOneSiteWeibo(restTemplate, name, author, desc, "酸味书屋", bookNum, href);
|
||||
log.debug(excuteNum + ":" + name + ":" + resultCode);
|
||||
log.debug(excuteNum + ":resultCode=={\"code\":\"A00006\"}" + "{\"code\":\"A00006\"}".equals(resultCode));
|
||||
|
||||
if ("{\"code\":\"A00006\"}".equals(resultCode)) {
|
||||
cacheUtil.set(BOOKNAME_CACHE_PREFIX + name, name, 60 * 60 * 24 * 30);
|
||||
}
|
||||
long realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000;
|
||||
Thread.sleep(realSleepMillis);
|
||||
}
|
||||
|
||||
}
|
||||
isFind = match.find();
|
||||
}
|
||||
|
||||
|
||||
String forObject2 = forEntity2.getBody();
|
||||
pattern = Pattern.compile("<div class=\"bookinfo\">" +
|
||||
"\\s*<a href=\"/(\\d*_\\d*)\\.html\">" +
|
||||
"\\s*<div class=\"detail\">" +
|
||||
"\\s*<p class=\"title\">(.*)</p>" +
|
||||
"\\s*<p class=\"author\">(.*)</p>" +
|
||||
"\\s*</div>" +
|
||||
"\\s*<div class=\"score\">(\\d*\\.\\d*)分</div>" +
|
||||
"\\s*</a>" +
|
||||
"\\s*</div>" +
|
||||
"\\s*<p class=\"review\"><span class=\"longview\"></span>((.*))</p>\\s*" +
|
||||
"</div>");
|
||||
match = pattern.matcher(forObject2);
|
||||
isFind = match.find();
|
||||
if (isFind) {
|
||||
while (isFind) {
|
||||
|
||||
float score = Float.parseFloat(match.group(4));
|
||||
|
||||
if (score >= 7.0) {
|
||||
|
||||
String bookNum = match.group(1);
|
||||
String href = "http://m.zinglizingli.xyz/" + bookNum + ".html";
|
||||
String name = match.group(2);
|
||||
if (!name.equals(cacheUtil.get(BOOKNAME_CACHE_PREFIX + name))) {
|
||||
String author = match.group(3);
|
||||
String desc = match.group(5);
|
||||
|
||||
String resultCode = sendOneSiteWeibo(restTemplate, name, author, desc, "喜羊羊小说网", bookNum, href);
|
||||
if ("{\"code\":\"A00006\"}".equals(resultCode)) {
|
||||
cacheUtil.set(BOOKNAME_CACHE_PREFIX + name, name, 60 * 60 * 24 * 30);
|
||||
}
|
||||
|
||||
long realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000;
|
||||
Thread.sleep(realSleepMillis);
|
||||
|
||||
}
|
||||
}
|
||||
isFind = match.find();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(),e);
|
||||
|
||||
} finally {
|
||||
isExcuting = false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
*/
|
||||
//19点到23点，1点到4点每隔50分钟执行一次,20本书*2分钟+空闲时间
|
||||
//@Scheduled(cron = "0 */50 19-23,1-4 * * ?")
|
||||
|
||||
/* public void sendAtNight() throws InterruptedException, IOException {
|
||||
if (!isExcuting) {
|
||||
isExcuting = true;
|
||||
log.info("sendAtNight定时器开始执行。。。。");
|
||||
long sleepMillis = 1000 * 60 * 2;
|
||||
sendAllSiteWeibo(sleepMillis);
|
||||
Thread.sleep(1000 * 60 * 10);
|
||||
isExcuting = false;
|
||||
}
|
||||
|
||||
|
||||
}*/
|
||||
|
||||
|
||||
//6点到17点每隔1小时执行一次，20本书*5分钟+空闲时间
|
||||
//@Scheduled(cron = "0 0 6-17/1 * * ?")
|
||||
/* public void sendAtDayTime() throws InterruptedException, IOException {
|
||||
|
||||
if (!isExcuting) {
|
||||
isExcuting = true;
|
||||
log.info("sendAtDayTime定时器开始执行。。。。");
|
||||
long sleepMillis = 1000 * 60 * 5;
|
||||
sendAllSiteWeibo(sleepMillis);
|
||||
Thread.sleep(1000 * 60 * 10);
|
||||
isExcuting = false;
|
||||
}
|
||||
|
||||
}*/
|
||||
|
||||
/*private void sendAllSiteWeibo(long sleepMillis) throws InterruptedException {
|
||||
RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");
|
||||
//分享酸味书屋
|
||||
String url = "http://www.zinglizingli.xyz/paihangbang_lastupdate/1.html";
|
||||
|
||||
//分享喜羊羊小说网
|
||||
String url2 = "http://m.zinglizingli.xyz/class/0/1.html";
|
||||
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(url, String.class);
|
||||
ResponseEntity<String> forEntity2 = restTemplate.getForEntity(url2, String.class);
|
||||
|
||||
String forObject = forEntity.getBody();
|
||||
Pattern pattern = Pattern.compile("<li class=\"tjxs\">\n" +
|
||||
"<span class=\"xsm\"><a href=\"/(\\d*_\\d*)/\">(.*)</a></span>\n" +
|
||||
"<span class=\"\">(.*)</span>\n" +
|
||||
"<span class=\"\">(.*)\\.\\.\\.</span>\n" +
|
||||
"<span class=\"tjrs\"><i>(\\d*)</i>人在看</span>\n" +
|
||||
"</li>");
|
||||
Matcher match = pattern.matcher(forObject);
|
||||
boolean isFind = match.find();
|
||||
if (isFind) {
|
||||
while (isFind) {
|
||||
|
||||
int lookNum = Integer.parseInt(match.group(5));
|
||||
if (lookNum > 5000) {
|
||||
String bookNum = match.group(1);
|
||||
String href = "http://www.zinglizingli.xyz/" + bookNum;
|
||||
String name = match.group(2);
|
||||
String author = match.group(3);
|
||||
String desc = match.group(4);
|
||||
sendOneSiteWeibo(restTemplate, name, author, desc, "酸味书屋", bookNum, href);
|
||||
long realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000;
|
||||
Thread.sleep(realSleepMillis);
|
||||
}
|
||||
|
||||
isFind = match.find();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
String forObject2 = forEntity2.getBody();
|
||||
pattern = Pattern.compile("<div class=\"bookinfo\">" +
|
||||
"\\s*<a href=\"/(\\d*_\\d*)/\">" +
|
||||
"\\s*<div class=\"detail\">" +
|
||||
"\\s*<p class=\"title\">(.*)</p>" +
|
||||
"\\s*<p class=\"author\">(.*)</p>" +
|
||||
"\\s*</div>" +
|
||||
"\\s*<div class=\"score\">(\\d*\\.\\d*)分</div>" +
|
||||
"\\s*</a>" +
|
||||
"\\s*</div>" +
|
||||
"\\s*<p class=\"review\"><span class=\"longview\"></span>((.*))</p>\\s*" +
|
||||
"</div>");
|
||||
match = pattern.matcher(forObject2);
|
||||
isFind = match.find();
|
||||
if (isFind) {
|
||||
while (isFind) {
|
||||
|
||||
float score = Float.parseFloat(match.group(4));
|
||||
|
||||
if (score >= 7.0) {
|
||||
|
||||
String bookNum = match.group(1);
|
||||
String href = "http://m.zinglizingli.xyz/" + bookNum;
|
||||
String name = match.group(2);
|
||||
String author = match.group(3);
|
||||
String desc = match.group(5);
|
||||
|
||||
sendOneSiteWeibo(restTemplate, name, author, desc, "喜羊羊小说网", bookNum, href);
|
||||
|
||||
long realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000;
|
||||
Thread.sleep(realSleepMillis);
|
||||
}
|
||||
|
||||
isFind = match.find();
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");
|
||||
//分享酸味书屋
|
||||
String url = "http://www.zinglizingli.xyz/paihangbang_lastupdate/1.html";
|
||||
ResponseEntity<String> forEntity = restTemplate.getForEntity(url, String.class);
|
||||
String forObject = forEntity.getBody();
|
||||
Pattern pattern = Pattern.compile("<li class=\"tjxs\">\n" +
|
||||
"<span class=\"xsm\"><a href=\"/(\\d*_\\d*)/\">(.*)</a></span>\n" +
|
||||
"<span class=\"\">(.*)</span>\n" +
|
||||
"<span class=\"\">(.*)\\.\\.\\.</span>\n" +
|
||||
"<span class=\"tjrs\"><i>\\d*</i>人在看</span>\n" +
|
||||
"</li>");
|
||||
Matcher match = pattern.matcher(forObject);
|
||||
boolean isFind = match.find();
|
||||
if (isFind) {
|
||||
while (isFind) {
|
||||
String bookNum = match.group(1);
|
||||
String href = "http://www.zinglizingli.xyz/" + bookNum;
|
||||
String name = match.group(2);
|
||||
String author = match.group(3);
|
||||
String desc = match.group(4);
|
||||
|
||||
|
||||
isFind = match.find();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
//分享喜羊羊小说网
|
||||
url = "http://m.zinglizingli.xyz/class/0/1.html";
|
||||
forEntity = restTemplate.getForEntity(url, String.class);
|
||||
forObject = forEntity.getBody();
|
||||
pattern = Pattern.compile("<div class=\"bookinfo\">" +
|
||||
"\\s*<a href=\"/(\\d*_\\d*)/\">" +
|
||||
"\\s*<div class=\"detail\">" +
|
||||
"\\s*<p class=\"title\">(.*)</p>" +
|
||||
"\\s*<p class=\"author\">(.*)</p>" +
|
||||
"\\s*</div>" +
|
||||
"\\s*<div class=\"score\">\\d*\\.\\d*分</div>" +
|
||||
"\\s*</a>" +
|
||||
"\\s*</div>" +
|
||||
"\\s*<p class=\"review\"><span class=\"longview\"></span>((.*))</p>\\s*" +
|
||||
"</div>");
|
||||
match = pattern.matcher(forObject);
|
||||
isFind = match.find();
|
||||
if (isFind) {
|
||||
while (isFind) {
|
||||
String bookNum = match.group(1);
|
||||
String href = "http://m.zinglizingli.xyz/" + bookNum;
|
||||
String name = match.group(2);
|
||||
String author = match.group(3);
|
||||
String desc = match.group(4);
|
||||
// sendOneSiteWeibo(restTemplate, name, author, desc, "喜羊羊小说网", bookNum, href);
|
||||
|
||||
|
||||
isFind = match.find();
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
private String sendOneSiteWeibo(RestTemplate template, String bookName, String indexName, String author, String desc, String wapName, String bookNum, String href) {
|
||||
String baseUrl = "http://service.weibo.com/share/aj_share.php";
|
||||
Map<String, String> param = new HashMap<>();
|
||||
/*String content = bookName + "小说最新章节列表," + bookName + "小说免费在线阅读," + bookName +
|
||||
"小说TXT下载,尽在" + wapName +href+ "\n";
|
||||
if(indexName != null){
|
||||
content+=("最新章节："+indexName+"\n");
|
||||
}
|
||||
content = content + "作者："+(author.replace("作者：","")) + "\n";
|
||||
content += ("简介："+desc.replace("简介：",""));*/
|
||||
String content = bookName+"最新章节,小说"+bookName+"("+author.replace("作者：","")+")手机阅读,小说"+bookName+"TXT下载 - "+href;
|
||||
param.put("content", content );
|
||||
param.put("styleid", "1");
|
||||
param.put("from", "share");
|
||||
param.put("appkey", "2351975812");
|
||||
param.put("refer", "http://www.zinglizingli.xyz/" + bookNum + "/");
|
||||
param.put("url_type", "0");
|
||||
param.put("visible", "0");
|
||||
//byte[] bytes = sendPostRequest(baseUrl, param);
|
||||
|
||||
|
||||
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
|
||||
map.setAll(param);
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add("Accept", "*/*");
|
||||
headers.add("Accept-Encoding", "gzip, deflate");
|
||||
headers.add("Accept-Language", "zh-CN,zh;q=0.9");
|
||||
headers.add("Connection", "keep-alive");
|
||||
headers.add("Content-Length", "1146");
|
||||
headers.add("Content-Type", "application/x-www-form-urlencoded");
|
||||
|
||||
String[] cookieArr = cookieStr.split(";");
|
||||
List<String> cookies = Arrays.asList(cookieArr);
|
||||
headers.put(HttpHeaders.COOKIE, cookies);
|
||||
|
||||
headers.add("Host", "service.weibo.com");
|
||||
headers.add("Origin", "http://service.weibo.com");
|
||||
headers.add("Referer", "http://service.weibo.com/share/share.php?appkey=2351975812&searchPic=true&title=%C2%A1%C2%BE%E4%BF%AE%E7%9C%9F%E8%81%8A%E5%A4%A9%E7%BE%A4%E6%9C%80%E6%96%B0%E7%AB%A0%E8%8A%82%E5%88%97%E8%A1%A8_%E4%BF%AE%E7%9C%9F%E8%81%8A%E5%A4%A9%E7%BE%A4%E6%9C%80%E6%96%B0%E7%AB%A0%E8%8A%82%E7%9B%AE%E5%BD%95_%E9%85%B8%E5%91%B3%E4%B9%A6%E5%B1%8B%C2%A1%C2%BF%E4%BF%AE%E7%9C%9F%E8%81%8A%E5%A4%A9%E7%BE%A4%E6%9C%80%E6%96%B0%E7%AB%A0%E8%8A%82%E7%94%B1%E7%BD%91%E5%8F%8B%E6%8F%90%E4%BE%9B%EF%BC%8C%E3%80%8A%E4%BF%AE%E7%9C%9F%E8%81%8A%E5%A4%A9%E7%BE%A4%E3%80%8B%E6%83%85%E8%8A%82%E8%B7%8C%E5%AE%95%E8%B5%B7%E4%BC%8F%E3%80%81%E6%89%A3%E4%BA%BA%E5%BF%83%E5%BC%A6%EF%BC%8C%E6%98%AF%E4%B8%80%E6%9C%AC%E6%83%85%E8%8A%82%E4%B8%8E%E6%96%87%E7%AC%94%E4%BF%B1%E4%BD%B3%E7%9A%84%E9%83%BD%E5%B8%82%E5%B0%8F%E8%AF%B4%E5%B0%8F%E8%AF%B4%EF%BC%8C%E9%85%B8%E5%91%B3%E4%B9%A6%E5%B1%8B%E5%85%8D%E8%B4%B9%E6%8F%90%E4%BE%9B%E5%94%90%E7%A0%96%E6%9C%80%E6%96%B0%E6%B8%85%E7%88%BD%E5%B9%B2%E5%87%80%E7%9A%84%E6%96%87%E5%AD%97%E7%AB%A0%E8%8A%82%E5%9C%A8%E7%BA%BF%E9%98%85%E8%AF%BB.&url=http%3A//www.zinglizingli.xyz/" + bookNum + "/");
|
||||
headers.add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE");
|
||||
headers.add("X-Requested-With", "XMLHttpRequest");
|
||||
headers.add("Accept-Encoding", "gzip, deflate");
|
||||
headers.add("Accept-Language", "zh-CN,zh;q=0.9");
|
||||
headers.add("Connection", "keep-alive");
|
||||
headers.add("Content-Length", "1146");
|
||||
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
|
||||
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
|
||||
|
||||
ResponseEntity<String> stringResponseEntity = template.postForEntity(baseUrl, request, String.class, map);
|
||||
|
||||
return stringResponseEntity.getBody();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
package xyz.zinglizingli.common.utils;
|
||||
|
||||
/**
|
||||
* @author XXY
|
||||
*/
|
||||
public class Constants {
|
||||
|
||||
/**
|
||||
* 服务异常路径
|
||||
* */
|
||||
public static final String SERVER_ERROR_PATH = "/mang.html";
|
||||
|
||||
/**
|
||||
* 404跳转路径
|
||||
* */
|
||||
public static final String NOT_FOUND_PATH = "/";
|
||||
|
||||
/**
|
||||
* 本地图片保存前缀
|
||||
* */
|
||||
public static final String LOCAL_PIC_PREFIX = "/localPic/";
|
||||
|
||||
/**
|
||||
* 漫画内容访问前缀
|
||||
* */
|
||||
public static final String CARTOON_CONTENT_PREFIX = "/manhua/";
|
||||
|
||||
/**
|
||||
* 漫画图片访问前缀
|
||||
* */
|
||||
public static final String CARTOON_PIC_PREFIX = "/manhua/images/";
|
||||
|
||||
/**
|
||||
* 漫画图片访问前缀
|
||||
* */
|
||||
public static final String CARTOON_STATIC_PREFIX = "/manhua/statics/";
|
||||
|
||||
/**
|
||||
* 爬取的动漫网站访问前缀
|
||||
* */
|
||||
public static final String CRAWL_CARTOON_URL_PREFIX = "https://www.dmzj.com/";
|
||||
|
||||
/**
|
||||
* 爬取的动漫网站图片访问前缀
|
||||
* */
|
||||
public static final String CRAWL_CARTOON_PIC_URL_PREFIX = "https://images.dmzj.com/";
|
||||
|
||||
/**
|
||||
* 爬取的动漫网站静态文件访问前缀
|
||||
* */
|
||||
public static final String CRAWL_CARTOON_STATIC_URL_PREFIX = "https://static.dmzj.com/";
|
||||
|
||||
|
||||
}
|
||||
@ -1,65 +0,0 @@
|
||||
package xyz.zinglizingli.common.utils;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class ContentFactory {
|
||||
|
||||
|
||||
private static String[] hotWords = {"雪鹰领主是我吃西红柿写作的一本非常经典的玄幻小说,本站免费提供雪鹰领主最新最全的TXT全文小说阅读。",
|
||||
"飞剑问道是我吃西红柿的经典仙侠小说作品，本站提供我吃西红柿TXT小说免费阅读。","" +
|
||||
"终极美女保镖小说阅读，《终极美女保镖》情节跌宕起伏、扣人心弦，是一本情节与文笔俱佳的玄幻小说，本站免费提供终极美女保镖最新清爽干净的文字章节在线阅读。",
|
||||
"一世倾城：冷宫弃妃,一世倾城：冷宫弃妃小说无弹窗阅读,一世倾城：冷宫弃妃小说无广告全文阅读,作者冷青衫",
|
||||
"校花之贴身高手最新章节，VIP章节免费阅读，校花之贴身高手由寂无倾情打造。看精品小说，上看小说吧！",
|
||||
"邪王追妻最新章节_邪王追妻免费全文阅读",
|
||||
"《最强反派系统》免费阅读，最强反派系统小说最新章节免费阅读，VIP章节免费阅读。",
|
||||
"大妖尊小说试阅读,大妖尊,大妖尊全文阅读,大妖尊最新章节",
|
||||
"诡秘之主，诡秘之主小说阅读。玄幻小说诡秘之主由作家爱潜水的乌贼创作，看小说吧提供诡秘之主首发最新章节及章节列表，诡秘之主最新更新尽在看小说吧",
|
||||
"全球高武，全球高武小说免费阅读。都市小说全球高武由作家老鹰吃小鸡创作，本站提供全球高武小说首发最新免费章节及章节列表，全球高武小说最新更新免费阅读","" +
|
||||
"前任无双由作家跃千愁创作，前任无双小说免费阅读，无双首发最新章节及章节列表免费阅读，无弹窗、无广告小说免费月的",
|
||||
"万古第一神小说是由作家风青阳所著的东方玄幻小说，本站提供万古第一神小说小说最新章节免费阅读",
|
||||
"《剑来》小说是烽火戏诸侯在著作的武侠仙侠, 本站提供无广告、无弹窗的剑来小说最新章节全文免费阅读,请随时关注更新最快的小说阅读网看小说吧。",
|
||||
"免费小说阅读,精彩免费小说最新章节尽在看小说吧,小说,小说网；网络小说；小说下载；小说txt,小说全文阅读，无弹窗广告尽在本站",
|
||||
"看小说吧收集、整理和免费分享作者几人哀愁的最新免费小说：回到过去当女神最新目录列表，回到过去当女神小说是属于网友受欢迎的奇幻玄幻类型的小说。",
|
||||
"大主宰无弹窗无广告最新免费章节由网友提供，《大主宰》小说情节跌宕起伏、扣人心弦：是一本情节与文笔俱佳的玄幻小说，看小说吧小说网免费大主宰最新免费的清爽干净的文字VIP章节在线阅读。",
|
||||
"旋风少女小说最新免费章节由网友提供，《旋风少女》是一本受欢迎的情节和文笔俱佳的都市小说，旋风少女无弹窗无广告小说最新章节目录免费阅读，旋风少女VIP章节免费阅读！",
|
||||
"斗破苍穹最新章节无弹窗是天蚕土豆倾力打造的一本非常耐读的小说，情节波澜起伏，由浅入深，层层推进，希望你能喜欢本书，支持天蚕土豆请收藏并推荐，斗破苍穹无弹窗最新免费章节目录免费提供"
|
||||
,"九星霸体诀是平凡魔术师写作的一本非常经典的玄幻小说,本站免费提供平凡魔术师最新最全的TXT全文小说阅读。",
|
||||
"万古神帝是飞天鱼的经典仙侠小说作品，本站提供万古神帝TXT小说免费阅读。","" +
|
||||
"无敌真寂寞小说阅读，《无敌真寂寞》情节跌宕起伏、扣人心弦，是一本情节与文笔俱佳的玄幻小说，本站免费提供无敌真寂寞最新清爽干净的文字章节在线阅读。",
|
||||
"绝鼎丹尊,绝鼎丹尊小说无弹窗阅读,绝鼎丹尊小说无广告全文阅读,作者万古青莲",
|
||||
"龙王传说最新章节，VIP章节免费阅读，龙王传说由唐家三少打造。看精品小说，上看小说吧！",
|
||||
"人道至尊，人道至尊小说阅读。玄幻小说人道至尊由作家宅猪创作，看小说吧提供人道至尊首发最新章节及章节列表，人道至尊最新更新尽在看小说吧小说网",
|
||||
"还是地球人狠，还是地球人狠免费阅读。都市小说全球高武由作家剑舞秀创作，本站提供还是地球人狠小说首发最新免费章节及章节列表，全球高武小说最新更新免费阅读","" +
|
||||
"《求魔》小说是在耳根著作的武侠仙侠, 本站提供无广告、无弹窗的求魔小说最新章节全文免费阅读,请随时关注更新最快的小说阅读网看小说吧。",
|
||||
"看小说吧收集、整理和免费分享作者唐家三少的最新免费小说：天火大道最新目录列表，天火大道小说是属于网友受欢迎的奇幻玄幻类型的小说。",
|
||||
"饲养全人类无弹窗无广告最新免费章节由网友提供，《饲养全人类》小说情节跌宕起伏、扣人心弦：是一本情节与文笔俱佳的玄幻小说，看小说吧小说网免费饲养全人类最新免费的清爽干净的文字VIP章节在线阅读。",
|
||||
"方外：消失的八门小说最新免费章节由网友提供，《方外：消失的八门》是一本受欢迎的情节和文笔俱佳的都市小说，方外：消失的八门无弹窗无广告小说最新章节目录免费阅读，方外：消失的八门VIP章节免费阅读！",
|
||||
"武林赘婿最新章节无弹窗是左山左行倾力打造的一本非常耐读的小说，情节波澜起伏，由浅入深，层层推进，希望你能喜欢本书，支持武林赘婿请收藏并推荐，武林赘婿无弹窗最新免费章节目录免费提供"};
|
||||
|
||||
|
||||
|
||||
public static Map<String,String> giveRandomContent(){
|
||||
Map<String,String> contentMap = new HashMap<>();
|
||||
int size = 0;
|
||||
|
||||
for(int i = 0 ; i < hotWords.length ; i++){
|
||||
if(size >= 18){
|
||||
break;
|
||||
}
|
||||
String value = hotWords[new Random().nextInt(hotWords.length)];
|
||||
if(contentMap.values().contains(value)){
|
||||
continue;
|
||||
}
|
||||
contentMap.put("hotWord_"+size,value);
|
||||
size++;
|
||||
}
|
||||
if(size < 18){
|
||||
for(int i = 0 ; i< 18 - size ; i++){
|
||||
contentMap.put("hotWord_"+size,hotWords[new Random().nextInt(hotWords.length)]);
|
||||
size++;
|
||||
}
|
||||
}
|
||||
return contentMap;
|
||||
|
||||
}
|
||||
}
|
||||
@ -6,23 +6,31 @@ import org.springframework.http.converter.StringHttpMessageConverter;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class RestTemplateUtil {
|
||||
|
||||
private static Map<String,RestTemplate> restTemplateMap = new HashMap<>();
|
||||
|
||||
public static RestTemplate getInstance(String charset) {
|
||||
HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory();
|
||||
httpRequestFactory.setConnectionRequestTimeout(3000);
|
||||
httpRequestFactory.setConnectTimeout(3000);
|
||||
httpRequestFactory.setReadTimeout(10000);
|
||||
RestTemplate restTemplate = new RestTemplate(httpRequestFactory);
|
||||
List<HttpMessageConverter<?>> list = restTemplate.getMessageConverters();
|
||||
for (HttpMessageConverter<?> httpMessageConverter : list) {
|
||||
if(httpMessageConverter instanceof StringHttpMessageConverter) {
|
||||
((StringHttpMessageConverter) httpMessageConverter).setDefaultCharset(Charset.forName(charset));
|
||||
break;
|
||||
public static RestTemplate getInstance(Charset charset) {
|
||||
RestTemplate restTemplate = restTemplateMap.get(charset.name());
|
||||
if(restTemplate == null) {
|
||||
|
||||
HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory();
|
||||
httpRequestFactory.setConnectionRequestTimeout(3000);
|
||||
httpRequestFactory.setConnectTimeout(3000);
|
||||
httpRequestFactory.setReadTimeout(10000);
|
||||
restTemplate = new RestTemplate(httpRequestFactory);
|
||||
List<HttpMessageConverter<?>> list = restTemplate.getMessageConverters();
|
||||
for (HttpMessageConverter<?> httpMessageConverter : list) {
|
||||
if (httpMessageConverter instanceof StringHttpMessageConverter) {
|
||||
((StringHttpMessageConverter) httpMessageConverter).setDefaultCharset(charset);
|
||||
break;
|
||||
}
|
||||
}
|
||||
restTemplateMap.put(charset.name(),restTemplate);
|
||||
}
|
||||
return restTemplate;
|
||||
}
|
||||
|
||||
@ -8,23 +8,17 @@ import org.springframework.stereotype.Component;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
|
||||
|
||||
/**
|
||||
* @author XXY
|
||||
*/
|
||||
@Component
|
||||
public class SpringUtil implements ApplicationContextAware {
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
String a = "<EFBFBD><EFBFBD><EFBFBD>";
|
||||
try {
|
||||
String b = new String(a.getBytes("ISO-8859-1"),"gbk");
|
||||
System.out.println(b);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static ApplicationContext applicationContext;
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
|
||||
if (SpringUtil.applicationContext == null) {
|
||||
SpringUtil.applicationContext = applicationContext;
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package xyz.zinglizingli.common.web;
|
||||
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.ui.ModelMap;
|
||||
@ -14,19 +15,21 @@ import xyz.zinglizingli.common.config.IndexRecBooksConfig;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 首页controller
|
||||
* @author XXY
|
||||
*/
|
||||
@Controller
|
||||
@RequestMapping
|
||||
@RequiredArgsConstructor
|
||||
public class IndexController {
|
||||
|
||||
|
||||
@Autowired
|
||||
private BookService bookService;
|
||||
private final BookService bookService;
|
||||
|
||||
@Autowired
|
||||
private CommonCacheUtil commonCacheUtil;
|
||||
private final CommonCacheUtil commonCacheUtil;
|
||||
|
||||
@Autowired
|
||||
private IndexRecBooksConfig indexRecBooksConfig;
|
||||
private final IndexRecBooksConfig indexRecBooksConfig;
|
||||
|
||||
|
||||
|
||||
@ -49,6 +52,7 @@ public class IndexController {
|
||||
hotBooks = bookService.search(1, 9, null, null, null, null, null, null, null, "visit_count DESC,score ", "DESC");
|
||||
commonCacheUtil.setObject(CacheKeyConstans.HOT_BOOK_LIST_KEY, hotBooks, 60 * 60 * 24);
|
||||
}
|
||||
|
||||
List<Book> newBooks = (List<Book>) commonCacheUtil.getObject(CacheKeyConstans.NEWST_BOOK_LIST_KEY);
|
||||
if (newBooks == null) {
|
||||
//查询最近更新数据
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user