From 289f1e16707b834d9644d8e6e1c124ab28cb20d8 Mon Sep 17 00:00:00 2001 From: xiongxiaoyang <773861846@qq.com> Date: Mon, 9 Dec 2019 16:17:59 +0800 Subject: [PATCH] =?UTF-8?q?=E9=98=BF=E9=87=8C=E5=B7=B4=E5=B7=B4=E8=A7=84?= =?UTF-8?q?=E8=8C=83=E6=94=B9=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- novel-front/novel-front.iml | 101 --- novel-front/pom.xml | 6 +- .../zinglizingli/books/mapper/BookMapper.java | 5 - .../books/service/BookService.java | 179 ++--- .../books/service/MailService.java | 50 +- .../books/service/UserService.java | 40 +- .../zinglizingli/books/util/ExcutorUtils.java | 3 + .../books/web/ApiBookController.java | 52 +- .../books/web/BookController.java | 89 ++- .../books/web/UserController.java | 40 +- .../common/config/ErrorConfig.java | 5 +- .../common/config/FilterConfig.java | 8 +- .../common/config/IndexRecBooksConfig.java | 4 + .../common/filter/BookFilter.java | 222 ++++++ .../common/filter/SearchFilter.java | 541 --------------- .../common/schedule/CrawlBooksSchedule.java | 642 +++++------------- .../common/schedule/SendEmaillSchedule.java | 56 -- .../common/schedule/SendUrlSchedule.java | 84 --- .../common/schedule/SendWeiboSchedule.java | 497 -------------- .../zinglizingli/common/utils/Constants.java | 54 ++ .../common/utils/ContentFactory.java | 65 -- .../common/utils/RestTemplateUtil.java | 30 +- .../zinglizingli/common/utils/SpringUtil.java | 14 +- .../common/web/IndexController.java | 16 +- 24 files changed, 785 insertions(+), 2018 deletions(-) create mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/filter/BookFilter.java delete mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java delete mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendEmaillSchedule.java delete mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendUrlSchedule.java delete mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendWeiboSchedule.java create mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/utils/Constants.java delete mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/utils/ContentFactory.java diff --git a/novel-front/novel-front.iml b/novel-front/novel-front.iml index c85b29f..eb5b6e4 100644 --- a/novel-front/novel-front.iml +++ b/novel-front/novel-front.iml @@ -5,105 +5,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/novel-front/pom.xml b/novel-front/pom.xml index d77fa74..94e48af 100644 --- a/novel-front/pom.xml +++ b/novel-front/pom.xml @@ -15,7 +15,6 @@ 小说精品楼-前台web网站 - UTF-8 UTF-8 1.8 @@ -69,6 +68,11 @@ 1.3.2 + + org.projectlombok + lombok + + diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java index 142ee95..2bc0c5a 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java @@ -35,11 +35,6 @@ public interface BookMapper { void addVisitCount(@Param("bookId") Long bookId); - Book queryRandomBook(); - - Book queryNewstBook(Set sendIds); - - List queryNewstBookIdList(); List queryEndBookIdList(); diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java b/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java index d8c28a9..d5902a4 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java @@ -1,20 +1,19 @@ package xyz.zinglizingli.books.service; -import com.fasterxml.jackson.databind.ObjectMapper; import com.github.pagehelper.PageHelper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.Charsets; +import org.apache.commons.lang3.StringUtils; import org.apache.http.client.utils.DateUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; -import org.springframework.http.*; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.util.StringUtils; -import org.springframework.web.client.RestTemplate; import tk.mybatis.orderbyhelper.OrderByHelper; import xyz.zinglizingli.books.constant.CacheKeyConstans; import xyz.zinglizingli.books.mapper.*; @@ -23,33 +22,31 @@ import xyz.zinglizingli.books.util.UUIDUtils; import xyz.zinglizingli.common.cache.CommonCacheUtil; import xyz.zinglizingli.common.utils.RestTemplateUtil; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +/** + * @author XXY + */ @Service +@RequiredArgsConstructor +@Slf4j public class BookService { - @Autowired - private BookMapper bookMapper; + private final BookMapper bookMapper; - @Autowired - private BookIndexMapper bookIndexMapper; + private final BookIndexMapper bookIndexMapper; - @Autowired - private BookContentMapper bookContentMapper; + private final BookContentMapper bookContentMapper; - @Autowired - private ScreenBulletMapper screenBulletMapper; + private final ScreenBulletMapper screenBulletMapper; - @Autowired - private UserRefBookMapper userRefBookMapper; + private final UserRefBookMapper userRefBookMapper; - @Autowired - private CommonCacheUtil cacheUtil; - - RestTemplate isoRestTemplate = RestTemplateUtil.getInstance("iso-8859-1"); + private final CommonCacheUtil cacheUtil; @Value("${pic.save.type}") private Byte picSaveType; @@ -58,16 +55,11 @@ public class BookService { private String picSavePath; - private Logger log = LoggerFactory.getLogger(BookService.class); - - + /** + * 保存章节目录和内容 + * */ public void saveBookAndIndexAndContent(Book book, List bookIndex, List bookContent){ - //一次最多只允许插入20条记录,否则影响服务器响应,如果没有插入所有更新，则更新时间设为昨天 - /*if(bookIndex.size()>100){ - book.setUpdateTime(new Date(book.getUpdateTime().getTime()-1000*60*60*24)); - } -*/ boolean isUpdate = false; Long bookId = -1L; @@ -81,12 +73,12 @@ public class BookService { bookId = books.get(0).getId(); book.setId(bookId); String picSrc = book.getPicUrl(); - if(picSaveType == 2 && org.apache.commons.lang3.StringUtils.isNotBlank(picSrc)){ + if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){ try { HttpHeaders headers = new HttpHeaders(); HttpEntity requestEntity = new HttpEntity<>(null, headers); - ResponseEntity resEntity = isoRestTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); - InputStream input = resEntity.getBody().getInputStream(); + ResponseEntity resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); + InputStream input = Objects.requireNonNull(resEntity.getBody()).getInputStream(); Date currentDate = new Date(); picSrc = "/localPic/" + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd") + UUIDUtils.getUUID32() @@ -127,12 +119,7 @@ public class BookService { } if (bookId >= 0) { - //查询目录已存在数量 - /* BookIndexExample bookIndexExample = new BookIndexExample(); - bookIndexExample.createCriteria().andBookIdEqualTo(bookId); - int indexCount = bookIndexMapper.countByExample(bookIndexExample);*/ - BookIndex lastIndex = null; List newBookIndexList = new ArrayList<>(); List newContentList = new ArrayList<>(); for (int i = 0; i < bookIndex.size(); i++) { @@ -141,11 +128,9 @@ public class BookService { BookIndex bookIndexItem = bookIndex.get(i); bookIndexItem.setBookId(bookId); bookContentItem.setBookId(bookId); - //bookContentItem.setIndexId(bookIndexItem.getId());暂时使用bookId和IndexNum查询content bookContentItem.setIndexNum(bookIndexItem.getIndexNum()); newBookIndexList.add(bookIndexItem); newContentList.add(bookContentItem); - lastIndex = bookIndexItem; } //一次最多只允许插入20条记录,否则影响服务器响应 if (isUpdate && i % 20 == 0 && newBookIndexList.size() > 0) { @@ -174,15 +159,21 @@ public class BookService { } - @Transactional + /** + * 批量插入章节目录表和章节内容表 + * */ + @Transactional(rollbackFor = Exception.class) public void insertIndexListAndContentList(List newBookIndexList, List newContentList) { bookIndexMapper.insertBatch(newBookIndexList); bookContentMapper.insertBatch(newContentList); } + /** + * 生成随机访问次数 + * */ private Long generateVisiteCount(Float score) { - int baseNum = (int) (Math.pow(score * 10, (int) (score - 5)) / 2); + int baseNum = (int)(score * 100); return Long.parseLong(baseNum + new Random().nextInt(1000) + ""); } @@ -202,12 +193,13 @@ public class BookService { OrderByHelper.orderBy(sortBy + " " + sort); } - List books = bookMapper.search(userId, ids, keyword, catId, softCat, softTag, bookStatus); - - return books; + return bookMapper.search(userId, ids, keyword, catId, softCat, softTag, bookStatus); } + /** + * 获取分类名 + * */ public String getCatNameById(Integer catid) { String catName = "其他"; @@ -257,11 +249,17 @@ public class BookService { return catName; } + /** + * 查询书籍的基础数据 + * */ public Book queryBaseInfo(Long bookId) { return bookMapper.selectByPrimaryKey(bookId); } + /** + * 查询最新更新的书籍列表 + * */ public List queryNewIndexList(Long bookId) { PageHelper.startPage(1, 15); BookIndexExample example = new BookIndexExample(); @@ -271,6 +269,9 @@ public class BookService { } + /** + * 查询书籍目录列表 + * */ public List queryAllIndexList(Long bookId) { BookIndexExample example = new BookIndexExample(); example.createCriteria().andBookIdEqualTo(bookId); @@ -278,6 +279,9 @@ public class BookService { return bookIndexMapper.selectByExample(example); } + /** + * 查询书籍章节内容 + * */ public BookContent queryBookContent(Long bookId, Integer indexNum) { BookContent content = (BookContent) cacheUtil.getObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum); if (content == null) { @@ -285,11 +289,6 @@ public class BookService { example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(indexNum); List bookContents = bookContentMapper.selectByExample(example); content = bookContents.size() > 0 ? bookContents.get(0) : null; - /*try { - content.setContent(chargeBookContent(content.getContent())); - } catch (IOException e) { - log.error(e.getMessage(), e); - }*/ cacheUtil.setObject(CacheKeyConstans.BOOK_CONTENT_KEY_PREFIX + "_" + bookId + "_" + indexNum, content, 60 * 60 * 24); } @@ -297,6 +296,9 @@ public class BookService { } + /** + * 增加访问次数 + * */ public void addVisitCount(Long bookId, String userId, Integer indexNum) { bookMapper.addVisitCount(bookId); @@ -307,13 +309,23 @@ public class BookService { } + /** + * 查询章节名 + * */ public String queryIndexNameByBookIdAndIndexNum(Long bookId, Integer indexNum) { BookIndexExample example = new BookIndexExample(); example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(indexNum); - return bookIndexMapper.selectByExample(example).get(0).getIndexName(); + List indexList = bookIndexMapper.selectByExample(example); + if(indexList != null && indexList.size() > 0 ) { + return indexList.get(0).getIndexName(); + } + return null; } + /** + * 查询最大和最小章节号 + * */ public List queryMaxAndMinIndexNum(Long bookId) { List result = new ArrayList<>(); BookIndexExample example = new BookIndexExample(); @@ -330,7 +342,7 @@ public class BookService { /** * 查询该书籍目录数量 */ - public List queryIndexCountByBookNameAndBAuthor(String bookName, String author) { + public List queryIndexCountByBookNameAndAuthor(String bookName, String author) { List result = new ArrayList<>(); BookExample example = new BookExample(); example.createCriteria().andBookNameEqualTo(bookName).andAuthorEqualTo(author); @@ -353,33 +365,11 @@ public class BookService { } - public Book queryRandomBook() { - return bookMapper.queryRandomBook(); - } - - public Map queryNewstBook() { - final String SENDIDS = "sendWeiboIds"; - Set sendIds = (Set) cacheUtil.getObject(SENDIDS); - if (sendIds == null) { - sendIds = new HashSet<>(); - } - String newstIndexName = null; - Book book = null; - book = bookMapper.queryNewstBook(sendIds); - Map data = new HashMap<>(); - if (book != null && book.getId() != null) { - newstIndexName = bookIndexMapper.queryNewstIndexName(book.getId()); - if (!StringUtils.isEmpty(newstIndexName)) { - sendIds.add(book.getId()); - cacheUtil.setObject(SENDIDS, sendIds, 60 * 60 * 24 * 2); - data.put("book", book); - data.put("newstIndexName", newstIndexName); - } - } - return data; - } + /** + * 查询轻小说分类名 + * */ public String getSoftCatNameById(Integer softCat) { String catName = "其他"; @@ -434,6 +424,9 @@ public class BookService { } + /** + * 查询漫画分类名 + * */ public String getMhCatNameById(Integer softCat) { String catName = "其他"; @@ -456,6 +449,9 @@ public class BookService { } + /** + * 保存弹幕 + * */ public void sendBullet(Long contentId, String bullet) { ScreenBullet screenBullet = new ScreenBullet(); @@ -466,6 +462,9 @@ public class BookService { screenBulletMapper.insertSelective(screenBullet); } + /** + * 查询弹幕 + * */ public List queryBullet(Long contentId) { ScreenBulletExample example = new ScreenBulletExample(); @@ -475,19 +474,19 @@ public class BookService { return screenBulletMapper.selectByExample(example); } - public String queryIndexList(Long bookId, int count) { - - BookIndexExample example = new BookIndexExample(); - example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(count); - return bookIndexMapper.selectByExample(example).get(0).getIndexName(); - } + /** + * 查询章节内容 + * */ public String queryContentList(Long bookId, int count) { BookContentExample example = new BookContentExample(); example.createCriteria().andBookIdEqualTo(bookId).andIndexNumEqualTo(count); return bookContentMapper.selectByExample(example).get(0).getContent(); } + /** + * 查询章节数 + * */ public int countIndex(Long bookId) { BookIndexExample example = new BookIndexExample(); example.createCriteria().andBookIdEqualTo(bookId); @@ -495,6 +494,9 @@ public class BookService { } + /** + * 查询完本书籍 + * */ public List queryEndBookIdList() { return bookMapper.queryEndBookIdList(); } @@ -503,6 +505,9 @@ public class BookService { + /** + * 查询前一章节和后一章节号 + * */ public List queryPreAndNextIndexNum(Long bookId, Integer indexNum) { List result = new ArrayList<>(); BookIndexExample example = new BookIndexExample(); diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/service/MailService.java b/novel-front/src/main/java/xyz/zinglizingli/books/service/MailService.java index 64af2e9..467196d 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/service/MailService.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/service/MailService.java @@ -1,8 +1,8 @@ package xyz.zinglizingli.books.service; +import lombok.RequiredArgsConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.FileSystemResource; import org.springframework.mail.SimpleMailMessage; @@ -14,36 +14,52 @@ import javax.mail.MessagingException; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; import java.io.File; -import java.io.UnsupportedEncodingException; +/** + * @author XXY + */ @Service +@RequiredArgsConstructor public class MailService { private final Logger logger = LoggerFactory.getLogger(MailService.class); + /** + * 使用@Value注入application.properties中指定的用户名 + * */ @Value("${spring.mail.username}") - //使用@Value注入application.properties中指定的用户名 private String from; - String nickName = "精品小说楼"; + private String nickName = "精品小说楼"; - @Autowired - //用于发送文件 - private JavaMailSender mailSender; + /** + * 用于发送文件 + * */ + private final JavaMailSender mailSender; + /** + * 发送简单邮件 + * */ public void sendSimpleMail(String to, String subject, String content) { SimpleMailMessage message = new SimpleMailMessage(); - message.setTo(to);//收信人 - message.setSubject(subject);//主题 - message.setText(content);//内容 - message.setFrom(from);//发信人 + //收信人 + message.setTo(to); + //主题 + message.setSubject(subject); + //内容 + message.setText(content); + //发信人 + message.setFrom(from); mailSender.send(message); } + /** + * 发送html邮件 + * */ public void sendHtmlMail(String to, String subject, String content){ logger.info("发送HTML邮件开始：{},{},{}", to, subject, content); @@ -57,7 +73,8 @@ public class MailService { helper.setFrom(new InternetAddress(from, nickName, "UTF-8")); helper.setTo(to); helper.setSubject(subject); - helper.setText(content, true);//true代表支持html + //true代表支持html + helper.setText(content, true); mailSender.send(message); logger.info("发送HTMLto"+to+"邮件成功"); } catch (Exception e) { @@ -65,6 +82,9 @@ public class MailService { } } + /** + * 发送带附件的邮件 + * */ public void sendAttachmentMail(String to, String subject, String content, String filePath) { logger.info("发送带附件邮件开始：{},{},{},{}", to, subject, content, filePath); @@ -80,7 +100,8 @@ public class MailService { helper.setText(content, true); FileSystemResource file = new FileSystemResource(new File(filePath)); String fileName = file.getFilename(); - helper.addAttachment(fileName, file);//添加附件，可多次调用该方法添加多个附件 + //添加附件，可多次调用该方法添加多个附件 + helper.addAttachment(fileName, file); mailSender.send(message); logger.info("发送带附件邮件成功"); } catch (MessagingException e) { @@ -103,7 +124,8 @@ public class MailService { helper.setSubject(subject); helper.setText(content, true); FileSystemResource res = new FileSystemResource(new File(rscPath)); - helper.addInline(rscId, res);//重复使用添加多个图片 + //重复使用添加多个图片 + helper.addInline(rscId, res); mailSender.send(message); logger.info("发送带图片邮件成功"); } catch (Exception e) { diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/service/UserService.java b/novel-front/src/main/java/xyz/zinglizingli/books/service/UserService.java index 9f8f765..d151c72 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/service/UserService.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/service/UserService.java @@ -1,7 +1,7 @@ package xyz.zinglizingli.books.service; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import xyz.zinglizingli.books.mapper.UserMapper; import xyz.zinglizingli.books.mapper.UserRefBookMapper; @@ -13,27 +13,38 @@ import xyz.zinglizingli.books.util.MD5Util; import java.util.List; +/** + * @author XXY + */ +@RequiredArgsConstructor @Service public class UserService { - @Autowired - private UserMapper userMapper; + private final UserMapper userMapper; - @Autowired - private UserRefBookMapper userRefBookMapper; + private final UserRefBookMapper userRefBookMapper; + /** + * 判断登录名是否存在 + * */ public boolean isExistLoginName(String loginName) { UserExample example = new UserExample(); example.createCriteria().andLoginNameEqualTo(loginName); - return userMapper.countByExample(example)>0?true:false; + return userMapper.countByExample(example)>0; } + /** + * 注册 + * */ public void regist(User user) { user.setPassword(MD5Util.MD5Encode(user.getPassword(),"utf-8")); userMapper.insertSelective(user); } + /** + * 登陆 + * */ public void login(User user) { UserExample example = new UserExample(); example.createCriteria().andLoginNameEqualTo(user.getLoginName()) @@ -48,6 +59,9 @@ public class UserService { } + /** + * 加入书架 + * */ public void addToCollect(Long bookId, long userId) { UserRefBook userRefBook = new UserRefBook(); userRefBook.setBookId(bookId); @@ -59,20 +73,29 @@ public class UserService { } + /** + * 判断是否加入书架 + * */ public boolean isCollect(Long bookId, long userId) { UserRefBookExample example = new UserRefBookExample(); example.createCriteria().andBookIdEqualTo(bookId).andUserIdEqualTo(userId); - return userRefBookMapper.countByExample(example)>0?true:false; + return userRefBookMapper.countByExample(example)>0; } + /** + * 取消加入书架 + * */ public void cancelToCollect(Long bookId, long userId) { UserRefBookExample example = new UserRefBookExample(); example.createCriteria().andBookIdEqualTo(bookId).andUserIdEqualTo(userId); userRefBookMapper.deleteByExample(example); } + /** + * 加入或取消书架 + * */ public void collectOrCancelBook(Long userid, Long bookId) { boolean collect = isCollect(bookId, userid); @@ -84,6 +107,9 @@ public class UserService { } } + /** + * 查询用户章节阅读记录 + * */ public Integer queryBookIndexNumber(String userId, Long bookId) { return userRefBookMapper.queryBookIndexNumber(userId,bookId); } diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/util/ExcutorUtils.java b/novel-front/src/main/java/xyz/zinglizingli/books/util/ExcutorUtils.java index 5e40951..0f5dd70 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/util/ExcutorUtils.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/util/ExcutorUtils.java @@ -3,6 +3,9 @@ package xyz.zinglizingli.books.util; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +/** + * @author XXY + */ public class ExcutorUtils { private static ExecutorService fixedThreadPool; diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/web/ApiBookController.java b/novel-front/src/main/java/xyz/zinglizingli/books/web/ApiBookController.java index cf754b9..d25824c 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/web/ApiBookController.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/web/ApiBookController.java @@ -2,10 +2,13 @@ package xyz.zinglizingli.books.web; import com.github.pagehelper.PageInfo; +import lombok.RequiredArgsConstructor; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.util.StringUtils; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; import xyz.zinglizingli.books.po.Book; import xyz.zinglizingli.books.po.BookContent; import xyz.zinglizingli.books.po.BookIndex; @@ -13,22 +16,26 @@ import xyz.zinglizingli.books.service.BookService; import xyz.zinglizingli.books.vo.BookVO; import xyz.zinglizingli.common.cache.CommonCacheUtil; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.util.*; +/** + * api接口 + * @author XXY + */ @RestController @RequestMapping("api/book") +@RequiredArgsConstructor public class ApiBookController { - @Autowired - private BookService bookService; + private final BookService bookService; - @Autowired - private CommonCacheUtil commonCacheUtil; + private final CommonCacheUtil commonCacheUtil; + /** + * 首页热门书籍查询接口 + * */ @RequestMapping("hotBook") public List hotBooks () { //查询热点数据 @@ -36,6 +43,9 @@ public class ApiBookController { return hotBooks; } + /** + * 首页最新书籍查询接口 + * */ @RequestMapping("newstBook") public List newstBook() { //查询最近更新数据 @@ -44,6 +54,9 @@ public class ApiBookController { return newBooks; } + /** + * 书籍搜索接口 + * */ @RequestMapping("search") public Map search(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize, @RequestParam(value = "keyword", required = false) String keyword, @@ -51,15 +64,14 @@ public class ApiBookController { @RequestParam(value = "catId", required = false) Integer catId, @RequestParam(value = "historyBookIds", required = false) String ids, @RequestParam(value = "token", required = false) String token, - @RequestParam(value = "sortBy", defaultValue = "update_time") String sortBy, @RequestParam(value = "sort", defaultValue = "DESC") String sort, - HttpServletRequest req, HttpServletResponse resp) { + @RequestParam(value = "sortBy", defaultValue = "update_time") String sortBy, @RequestParam(value = "sort", defaultValue = "DESC") String sort + ) { Map modelMap = new HashMap<>(); String userId = null; String titleType = "最近更新"; if (catId != null) { titleType = bookService.getCatNameById(catId); - ; } else if (keyword != null) { titleType = "搜索"; } else if ("score".equals(sortBy)) { @@ -114,6 +126,9 @@ public class ApiBookController { return modelMap; } + /** + * 书籍详情信息查询接口 + * */ @RequestMapping("{bookId}.html") public Map detail(@PathVariable("bookId") Long bookId) { Map modelMap = new HashMap<>(); @@ -131,6 +146,9 @@ public class ApiBookController { return modelMap; } + /** + * 书籍目录查询接口 + * */ @RequestMapping("{bookId}/index.html") public Map bookIndex(@PathVariable("bookId") Long bookId) { Map modelMap = new HashMap<>(); @@ -142,6 +160,9 @@ public class ApiBookController { return modelMap; } + /** + * 书籍章节内容查询接口 + * */ @RequestMapping("{bookId}/{indexNum}.html") public Map bookContent(@PathVariable("bookId") Long bookId, @PathVariable("indexNum") Integer indexNum) { Map modelMap = new HashMap<>(); @@ -161,7 +182,7 @@ public class ApiBookController { String indexName; if(bookContent==null) { bookContent = new BookContent(); - bookContent.setId(-1l); + bookContent.setId(-1L); bookContent.setBookId(bookId); bookContent.setIndexNum(indexNum); bookContent.setContent("正在手打中，请稍等片刻，内容更新后，需要重新刷新页面，才能获取最新更新"); @@ -176,13 +197,6 @@ public class ApiBookController { return modelMap; } - /*@RequestMapping("addVisit") - public String addVisit(@RequestParam("bookId") Long bookId) { - - bookService.addVisitCount(bookId, userId, indexNum); - - return "ok"; - }*/ } diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/web/BookController.java b/novel-front/src/main/java/xyz/zinglizingli/books/web/BookController.java index fbd6272..e1d08f9 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/web/BookController.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/web/BookController.java @@ -2,9 +2,9 @@ package xyz.zinglizingli.books.web; import com.github.pagehelper.PageInfo; +import lombok.RequiredArgsConstructor; import org.apache.commons.text.StringEscapeUtils; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.util.StringUtils; @@ -26,24 +26,30 @@ import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; import java.io.OutputStream; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.*; +/** + * 小说Controller + * @author 11797 + */ @Controller @RequestMapping("book") +@RequiredArgsConstructor public class BookController { - @Autowired - private BookService bookService; + private final BookService bookService; - @Autowired - private UserService userService; + private final UserService userService; - @Autowired - private CommonCacheUtil commonCacheUtil; + private final CommonCacheUtil commonCacheUtil; + /** + * 精品小说搜索页 + * */ @RequestMapping("search") public String search(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize, @RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "catId", required = false) Integer catId, @@ -120,6 +126,9 @@ public class BookController { } + /** + * 轻小说搜索页 + * */ @RequestMapping("searchSoftBook.html") public String searchSoftBook(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize, @RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "catId", defaultValue = "8") Integer catId, @@ -155,6 +164,9 @@ public class BookController { return "books/soft_book_search"; } + /** + * 漫画搜索页 + * */ @RequestMapping("searchMhBook.html") public String searchMhBook(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize, @RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "catId", defaultValue = "9") Integer catId, @@ -190,6 +202,9 @@ public class BookController { return "books/mh_book_search"; } + /** + * 书籍详情页 + * */ @RequestMapping("{bookId}.html") public String detail(@PathVariable("bookId") Long bookId, @RequestParam(value = "token",required = false)String token, ModelMap modelMap) { String userId = commonCacheUtil.get(token); @@ -229,6 +244,9 @@ public class BookController { return "books/book_detail"; } + /** + * 书籍目录页 + * */ @RequestMapping("{bookId}/index.html") public String bookIndex(@PathVariable("bookId") Long bookId, ModelMap modelMap) { List indexList = bookService.queryAllIndexList(bookId); @@ -239,13 +257,17 @@ public class BookController { return "books/book_index"; } + + /** + * 书籍内容页 + * */ @RequestMapping("{bookId}/{indexNum}.html") public String bookContent(@PathVariable("bookId") Long bookId, @PathVariable("indexNum") Integer indexNum, ModelMap modelMap) { BookContent bookContent = bookService.queryBookContent(bookId, indexNum); String indexName; if (bookContent == null) { bookContent = new BookContent(); - bookContent.setId(-1l); + bookContent.setId(-1L); bookContent.setBookId(bookId); bookContent.setIndexNum(indexNum); bookContent.setContent("正在手打中，请稍等片刻，内容更新后，需要重新刷新页面，才能获取最新更新"); @@ -271,6 +293,9 @@ public class BookController { } + /** + * 增加访问次数 + * */ @RequestMapping("addVisit") @ResponseBody public String addVisit(@RequestParam("bookId") Long bookId,@RequestParam(value = "indexNum",defaultValue = "0") Integer indexNum,@RequestParam(value = "token",defaultValue = "") String token) { @@ -281,6 +306,9 @@ public class BookController { return "ok"; } + /** + * 发送弹幕 + * */ @RequestMapping("sendBullet") @ResponseBody public Map sendBullet(@RequestParam("contentId") Long contentId, @RequestParam("bullet") String bullet) { @@ -291,6 +319,9 @@ public class BookController { return result; } + /** + * 查询是否正在下载 + * */ @RequestMapping("queryIsDownloading") @ResponseBody public Map queryIsDownloading(HttpSession session) { @@ -304,6 +335,9 @@ public class BookController { } + /** + * 查询弹幕 + * */ @RequestMapping("queryBullet") @ResponseBody public List queryBullet(@RequestParam("contentId") Long contentId) { @@ -325,27 +359,30 @@ public class BookController { OutputStream out = resp.getOutputStream(); //设置响应头，对文件进行url编码 bookName = URLEncoder.encode(bookName, "UTF-8"); - resp.setContentType("application/octet-stream");//解决手机端不能下载附件的问题 + //解决手机端不能下载附件的问题 + resp.setContentType("application/octet-stream"); resp.setHeader("Content-Disposition", "attachment;filename=" + bookName + ".txt"); if (count > 0) { for (int i = 0; i < count; i++) { - String index = bookService.queryIndexList(bookId, i); - String content = bookService.queryContentList(bookId, i); - out.write(index.getBytes("utf-8")); - out.write("\n".getBytes("utf-8")); - content = content.replaceAll("", "\r\n"); - content = content.replaceAll(" ", " "); - content = content.replaceAll("]*>", ""); - content = content.replaceAll("", ""); - content = content.replaceAll("]*>", ""); - content = content.replaceAll("", ""); - content = content.replaceAll("]*>[^<]*]*>[^<]*\\s*

", ""); - content = content.replaceAll("]*>", ""); - content = content.replaceAll("

", "\r\n"); - out.write(content.getBytes("utf-8")); - out.write("\r\n".getBytes("utf-8")); - out.write("\r\n".getBytes("utf-8")); - out.flush(); + String index = bookService.queryIndexNameByBookIdAndIndexNum(bookId, i); + if(index != null) { + String content = bookService.queryContentList(bookId, i); + out.write(index.getBytes(StandardCharsets.UTF_8)); + out.write("\n".getBytes(StandardCharsets.UTF_8)); + content = content.replaceAll("", "\r\n"); + content = content.replaceAll(" ", " "); + content = content.replaceAll("]*>", ""); + content = content.replaceAll("", ""); + content = content.replaceAll("]*>", ""); + content = content.replaceAll("", ""); + content = content.replaceAll("]*>[^<]*]*>[^<]*\\s*

", ""); + content = content.replaceAll("]*>", ""); + content = content.replaceAll("

", "\r\n"); + out.write(content.getBytes(StandardCharsets.UTF_8)); + out.write("\r\n".getBytes(StandardCharsets.UTF_8)); + out.write("\r\n".getBytes(StandardCharsets.UTF_8)); + out.flush(); + } } } diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/web/UserController.java b/novel-front/src/main/java/xyz/zinglizingli/books/web/UserController.java index 48eba05..0e8e006 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/web/UserController.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/web/UserController.java @@ -1,6 +1,7 @@ package xyz.zinglizingli.books.web; +import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; @@ -15,21 +16,23 @@ import xyz.zinglizingli.common.cache.CommonCacheUtil; import java.util.*; +/** + * @author XXY + */ @Controller @RequestMapping("user") +@RequiredArgsConstructor public class UserController { - @Autowired - private UserService userService; + private final UserService userService; - @Autowired - private BookService bookService; - - @Autowired - private CommonCacheUtil commonCacheUtil; + private final CommonCacheUtil commonCacheUtil; + /** + * 登陆页面 + * */ @RequestMapping("login.html") public String login(Long bookId, ModelMap modelMap) { modelMap.put("bookId", bookId); @@ -37,10 +40,13 @@ public class UserController { } + /** + * 登陆或注册 + * */ @RequestMapping("loginOrRegist") @ResponseBody public Map loginOrRegist(User user,Long bookId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(2); //查询用户名是否存在 boolean isExistLoginName = userService.isExistLoginName(user.getLoginName()); String token = null; @@ -77,10 +83,13 @@ public class UserController { return result; } + /** + * 登陆状态查询 + * */ @RequestMapping("isLogin") @ResponseBody public Map isLogin(String token) { - Map result = new HashMap<>(); + Map result = new HashMap<>(2); String userId = commonCacheUtil.get(token); if(StringUtils.isEmpty(userId)){ result.put("code", -1); @@ -93,6 +102,9 @@ public class UserController { } + /** + * 加入书架 + * */ @RequestMapping("addToCollect") @ResponseBody public Map addToCollect(Long bookId,String token) { @@ -109,10 +121,13 @@ public class UserController { return result; } + /** + * 撤下书架 + * */ @RequestMapping("cancelToCollect") @ResponseBody public Map cancelToCollect(Long bookId,String token) { - Map result = new HashMap<>(); + Map result = new HashMap<>(2); String userId = commonCacheUtil.get(token); if(StringUtils.isEmpty(userId)){ result.put("code", -1); @@ -125,10 +140,13 @@ public class UserController { return result; } + /** + * 判断是否加入书架 + * */ @RequestMapping("isCollect") @ResponseBody public Map isCollect(Long bookId,String token) { - Map result = new HashMap<>(); + Map result = new HashMap<>(2); String userId = commonCacheUtil.get(token); if(StringUtils.isEmpty(userId)){ result.put("code", -1); diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/config/ErrorConfig.java b/novel-front/src/main/java/xyz/zinglizingli/common/config/ErrorConfig.java index a765149..d8ebfaf 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/config/ErrorConfig.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/config/ErrorConfig.java @@ -5,6 +5,7 @@ import org.springframework.boot.web.server.ErrorPageRegistrar; import org.springframework.boot.web.server.ErrorPageRegistry; import org.springframework.context.annotation.Configuration; import org.springframework.http.HttpStatus; +import xyz.zinglizingli.common.utils.Constants; /** *定义配置类 @@ -15,8 +16,8 @@ public class ErrorConfig implements ErrorPageRegistrar { @Override public void registerErrorPages(ErrorPageRegistry registry) { ErrorPage[] errorPages = new ErrorPage[2]; - errorPages[0] = new ErrorPage(HttpStatus.NOT_FOUND, "/book/index.html"); - errorPages[1] = new ErrorPage(HttpStatus.INTERNAL_SERVER_ERROR, "/mang.html"); + errorPages[0] = new ErrorPage(HttpStatus.NOT_FOUND, Constants.NOT_FOUND_PATH); + errorPages[1] = new ErrorPage(HttpStatus.INTERNAL_SERVER_ERROR, Constants.SERVER_ERROR_PATH); registry.addErrorPages(errorPages); } diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java b/novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java index 65cc8f9..8654492 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java @@ -4,7 +4,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import xyz.zinglizingli.common.filter.SearchFilter; +import xyz.zinglizingli.common.filter.BookFilter; @Configuration public class FilterConfig{ @@ -13,9 +13,9 @@ public class FilterConfig{ private String picSavePath; @Bean - public FilterRegistrationBean filterRegist() { - FilterRegistrationBean frBean = new FilterRegistrationBean(); - frBean.setFilter(new SearchFilter()); + public FilterRegistrationBean filterRegist() { + FilterRegistrationBean frBean = new FilterRegistrationBean<>(); + frBean.setFilter(new BookFilter()); frBean.addUrlPatterns("/*"); frBean.addInitParameter("picSavePath",picSavePath); return frBean; diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/config/IndexRecBooksConfig.java b/novel-front/src/main/java/xyz/zinglizingli/common/config/IndexRecBooksConfig.java index 33088af..855d0d2 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/config/IndexRecBooksConfig.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/config/IndexRecBooksConfig.java @@ -6,6 +6,10 @@ import org.springframework.stereotype.Component; import java.util.List; import java.util.Map; +/** + * 首页推荐书籍配置 + * @author XXY + * */ @Component @ConfigurationProperties(prefix = "index") public class IndexRecBooksConfig { diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/filter/BookFilter.java b/novel-front/src/main/java/xyz/zinglizingli/common/filter/BookFilter.java new file mode 100644 index 0000000..ca991bf --- /dev/null +++ b/novel-front/src/main/java/xyz/zinglizingli/common/filter/BookFilter.java @@ -0,0 +1,222 @@ +package xyz.zinglizingli.common.filter; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.Charsets; +import org.springframework.core.io.Resource; +import org.springframework.http.*; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; +import xyz.zinglizingli.common.utils.Constants; +import xyz.zinglizingli.common.utils.RestTemplateUtil; + +import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.*; + +/** + * 书籍内容过滤器 + * @author xxy + */ +@Slf4j +public class BookFilter implements Filter { + + + /** + * 图片后缀集合 + * */ + private final List PIC_POSTFIX_LIST = Arrays.asList("jpg","pcx","emf","gif","bmp","tga","jpeg","tif","png","rle"); + + /** + * 本地图片保存路径前缀 + * */ + private String picSavePath; + + + @Override + public void init(FilterConfig filterConfig){ + picSavePath = filterConfig.getInitParameter("picSavePath"); + } + + @SneakyThrows + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain){ + HttpServletRequest req = (HttpServletRequest) servletRequest; + HttpServletResponse resp = (HttpServletResponse) servletResponse; + String requestUrl = req.getRequestURL().toString(); + String requestUri = req.getRequestURI(); + + String forObject; + + try { + + //本地图片 + if (requestUri.contains(Constants.LOCAL_PIC_PREFIX)) { + OutputStream out = resp.getOutputStream(); + InputStream input = new FileInputStream(new File(picSavePath + requestUri)); + byte[] b = new byte[4096]; + for (int n; (n = input.read(b)) != -1; ) { + out.write(b, 0, n); + } + input.close(); + out.close(); + return; + + } + + //非漫画访问 + if (!requestUrl.contains(Constants.CARTOON_CONTENT_PREFIX)) { + filterChain.doFilter(req, resp); + return; + } + + + String method = req.getMethod(); + + //请求html/css/js等文件 + RestTemplate restTemplate = RestTemplateUtil.getInstance(Charsets.UTF_8); + + resp.setCharacterEncoding("utf-8"); + + String realPath = Constants.CRAWL_CARTOON_URL_PREFIX + requestUri.substring(Constants.CARTOON_CONTENT_PREFIX.length()); + + + if (HttpMethod.GET.name().equals(method)) { + //漫画GET请求 + String postFix = requestUri.substring(requestUri.lastIndexOf(".") + 1); + if (PIC_POSTFIX_LIST.contains(postFix)) { + // 请求图片 + restTemplate = RestTemplateUtil.getInstance(Charsets.ISO_8859_1); + resp.setContentType("image/apng"); + } + if (requestUri.startsWith(Constants.CARTOON_PIC_PREFIX)) { + //漫画图片处理 + HttpHeaders headers = new HttpHeaders(); + headers.add("Referer", Constants.CRAWL_CARTOON_URL_PREFIX); + HttpEntity requestEntity = new HttpEntity<>(null, headers); + realPath = Constants.CRAWL_CARTOON_PIC_URL_PREFIX + requestUri.substring(Constants.CARTOON_PIC_PREFIX.length()); + ResponseEntity resEntity = restTemplate.exchange(realPath, HttpMethod.GET, requestEntity, Resource.class); + InputStream input = Objects.requireNonNull(resEntity.getBody()).getInputStream(); + OutputStream out = resp.getOutputStream(); + byte[] b = new byte[4096]; + for (int n; (n = input.read(b)) != -1; ) { + out.write(b, 0, n); + } + input.close(); + out.close(); + return; + + + } else if (requestUri.startsWith(Constants.CARTOON_STATIC_PREFIX)) { + //漫画静态文件处理 + realPath = Constants.CRAWL_CARTOON_STATIC_URL_PREFIX + requestUri.substring(Constants.CARTOON_STATIC_PREFIX.length()); + ResponseEntity forEntity = restTemplate.getForEntity(realPath, String.class); + forObject = forEntity.getBody(); + assert forObject != null; + forObject = forObject.replaceAll(Constants.CRAWL_CARTOON_PIC_URL_PREFIX, Constants.CARTOON_PIC_PREFIX); + + } else { + //其他漫画内容处理 + ResponseEntity forEntity = restTemplate.getForEntity(realPath, String.class); + forObject = forEntity.getBody(); + + assert forObject != null; + + //漫画内容过滤 + forObject = forObject.replace(Constants.CRAWL_CARTOON_URL_PREFIX+"js/ad/ad_12.js", "") + .replace(Constants.CRAWL_CARTOON_URL_PREFIX+"js/ad/ad_13.js", "") + .replace(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX+"ocomic/js/dmzjMhFinally-new.js", "") + .replace(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX+"ocomic/js/dmzjMhFinally-new.js", "") + .replace(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX+"module/js/float_code.js", "") + .replaceAll(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX+"public/js/dmzj-land-2015.6.js", "") + .replaceAll("", "") + .replaceAll("globalNav.js", "") + .replaceAll("TSB.js", "") + .replaceAll(Constants.CRAWL_CARTOON_PIC_URL_PREFIX, Constants.CARTOON_PIC_PREFIX) + .replaceAll(Constants.CRAWL_CARTOON_STATIC_URL_PREFIX, Constants.CARTOON_STATIC_PREFIX) + + "" + + ""; + } + + + } else { + //漫画POST请求 + + + Map oldParameterMap = req.getParameterMap(); + Set> entries = oldParameterMap.entrySet(); + Map newParameterMap = new HashMap<>(entries.size()); + for (Map.Entry entry : entries) { + newParameterMap.put(entry.getKey(), entry.getValue()[0]); + } + + MultiValueMap map = new LinkedMultiValueMap<>(); + map.setAll(newParameterMap); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + HttpEntity> request = new HttpEntity<>(map, headers); + forObject = restTemplate.postForEntity(realPath, request, String.class).getBody(); + assert forObject != null; + forObject = forObject.replaceAll(Constants.CARTOON_CONTENT_PREFIX, Constants.CRAWL_CARTOON_URL_PREFIX) + ; + + } + + + } catch (RuntimeException e) { + log.error(e.getMessage(), e); + if (e instanceof HttpClientErrorException && (((HttpClientErrorException) e).getStatusCode() == HttpStatus.NOT_FOUND)) { + //404 + resp.sendRedirect(Constants.NOT_FOUND_PATH); + return; + } else { + //500 + req.getRequestDispatcher(Constants.SERVER_ERROR_PATH).forward(servletRequest, servletResponse); + return; + } + + + } + + resp.getWriter().print(forObject); + } + + + @Override + public void destroy() { + + } + + +} diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java b/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java deleted file mode 100644 index 0cee3d1..0000000 --- a/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java +++ /dev/null @@ -1,541 +0,0 @@ -package xyz.zinglizingli.common.filter; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.Resource; -import org.springframework.http.*; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.client.HttpClientErrorException; -import org.springframework.web.client.RestTemplate; -import xyz.zinglizingli.common.cache.CommonCacheUtil; -import xyz.zinglizingli.common.utils.RestTemplateUtil; -import xyz.zinglizingli.common.utils.SpringUtil; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.*; -import java.net.URLDecoder; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class SearchFilter implements Filter { - - private static final Logger log = LoggerFactory.getLogger(SearchFilter.class); - - private CommonCacheUtil cacheUtil; - - private static List picPostFix; - - private static List localFileFix; - - private static List staticFileFix; - - private static List noteURI; - - private RestTemplate restTemplate; - - private String picSavePath; - - - - private final String SUANWEI_BOOK_REGEX = ""; - private final String SUANWEI_BOOK_HTML_REGEX = "/\\d+_\\d+\\.html"; - - private final String XIYANGYANG_BOOK_REGEX = ""; - private final String XIYANGYANG_BOOK_HTML_REGEX = "/\\d+_\\d+\\.html"; - - - @Override - public void init(FilterConfig filterConfig) throws ServletException { - picSavePath = filterConfig.getInitParameter("picSavePath"); - picPostFix = new ArrayList<>(); - picPostFix.add("jpg"); - picPostFix.add("pcx"); - picPostFix.add("emf"); - picPostFix.add("gif"); - picPostFix.add("bmp"); - picPostFix.add("tga"); - picPostFix.add("jpeg"); - picPostFix.add("tif"); - picPostFix.add("png"); - picPostFix.add("rle"); - localFileFix = new ArrayList<>(); - localFileFix.add("IMG_1470.JPG"); - localFileFix.add("baidu_verify_Ep8xaWQJAI.html"); - localFileFix.add("baidu_verify_L6sR9GjEtg.html"); - localFileFix.add("shenma-site-verification.txt"); - localFileFix.add("favicon.ico"); - localFileFix.add("headerbg.jpg"); - localFileFix.add("mang.png"); - localFileFix.add("HotBook.apk"); - localFileFix.add("wap_collect.js"); - localFileFix.add("note_1.html"); - localFileFix.add("note_2.html"); - localFileFix.add("note_3.html"); - localFileFix.add("note_4.html"); - staticFileFix = new ArrayList<>(); - staticFileFix.add("jpg"); - staticFileFix.add("gif"); - staticFileFix.add("bmp"); - staticFileFix.add("jpeg"); - staticFileFix.add("png"); - staticFileFix.add("js"); - staticFileFix.add("css"); - noteURI = new ArrayList<>(); - noteURI.add("/html/note_1.html"); - noteURI.add("/html/note_2.html"); - - } - - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - String forObject = null; - HttpServletRequest req = (HttpServletRequest) servletRequest; - HttpServletResponse resp = (HttpServletResponse) servletResponse; - String requestURL = req.getRequestURL().toString(); - String requestURI = req.getRequestURI(); - - try { - - if(requestURI.contains("/localPic/")){ - OutputStream out = resp.getOutputStream(); - InputStream input = new FileInputStream(new File(picSavePath+requestURI)); - byte[] b = new byte[4096]; - for (int n; (n = input.read(b)) != -1;) { - out.write(b, 0, n); - } - input.close(); - out.close(); - return; - - } - - if (!requestURL.contains("/manhua/")) { - filterChain.doFilter(req, resp); - return; - } - - - final String method = req.getMethod(); - - - // 案例：充当客户端通过restTemplate请求网络数据，并充当服务端将数据返回给浏览器 - // 客户端请求数据：输入流（byte[]）==》字符串 - // 服务端响应数据：字符串 == 》 输出流（byte[]） - - //默认方式： - //RestTemplate restTemplate = new RestTemplate(); - // ①当返回的response-header的content-type属性有charset值时， - // restTemplate的StringHttpMessageConverter会设置默认charset为content-type属性 - // charset值 - // StringHttpMessageConverter.setDefaultCharset(Charset.forName(charset)); - // ②当返回的response-header的content-type属性没有charset值时 - // restTemplate的StringHttpMessageConverter会使用默认的charset即ISO-8859-1 - - // 对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String） - restTemplate = RestTemplateUtil.getInstance("utf-8");//请求html/css/js等文件 - // 对客户端响应返回的字符串（String）采用何种编码转换成输出流（byte[]） - resp.setCharacterEncoding("utf-8"); - - /*//=====现在浏览器有编码自动识别功能，所以上面的代码没有加content-type的Header也没有问题========== - //=====正确做法应该是下面代码片段1和代码片段2二选一========== - - //===============================================代码片段1=============================== - // 对客户端响应返回的字符串（String）采用何种编码转换成输出流（byte[]） - resp.setCharacterEncoding("utf-8"); - // 告诉浏览器对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String）显示 - resp.setHeader("content-type", "text/html;charset=utf-8"); - //===============================================代码片段1=============================== - - - //===============================================代码片段2=============================== - //对客户端响应返回的字符串（String）采用何种编码转换成输出流（byte[]） - //并且告诉浏览器对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String）显示 - resp.setContentType("text/html;charset=utf-8"); - //===============================================代码片段2=============================== -*/ - - - if (HttpMethod.GET.name().equals(method)) { - - - String fileName = requestURI.substring(requestURI.lastIndexOf("/") + 1); - if (localFileFix.contains(fileName) || fileName.startsWith("9a4a540e-1759-4268-90fa-7fb652c3604a.")) { - filterChain.doFilter(servletRequest, servletResponse); - return; - } - - - String queryString = req.getQueryString(); - if (queryString != null && queryString.length() > 0 && !queryString.contains("bsh_bid=")) { - queryString = "?" + URLDecoder.decode(req.getQueryString()); - } else { - queryString = ""; - } - - - if (forObject == null) { - String postFix = requestURI.substring(requestURI.lastIndexOf(".") + 1); - if (picPostFix.contains(postFix)) { - // 对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String） - restTemplate = RestTemplateUtil.getInstance("iso-8859-1");//请求图片 - resp.setContentType("image/apng"); - } - if (requestURI.startsWith("/manhua/images/")) { - HttpHeaders headers = new HttpHeaders(); - headers.add("Referer","https://www.dmzj.com"); - HttpEntity requestEntity = new HttpEntity<>(null, headers); - String realUrl = "https://images.dmzj.com/" + requestURI.substring(15); - ResponseEntity resEntity = restTemplate.exchange(realUrl, HttpMethod.GET, requestEntity, Resource.class); - InputStream input = resEntity.getBody().getInputStream(); - OutputStream out = resp.getOutputStream(); - byte[] b = new byte[4096]; - for (int n; (n = input.read(b)) != -1;) { - out.write(b, 0, n); - } - input.close(); - out.close(); - return; - - - // 对服务端请求返回的输入流（byte[]）采用何种编码转换成字符串（String） - - - - - } else if(requestURI.startsWith("/manhua/statics/")){ - String realPath = "https://static.dmzj.com/"+requestURI.substring(16); - ResponseEntity forEntity = restTemplate.getForEntity(realPath, String.class); - forObject = forEntity.getBody(); - forObject = forObject.replaceAll("https://images.dmzj.com/", "/manhua/images/"); - - }else { - String realPath = "https://www.dmzj.com/" + requestURI.substring(8); - - ResponseEntity forEntity = restTemplate.getForEntity(realPath, String.class); - forObject = forEntity.getBody(); - - // forObject = new String(forObject.getBytes("ISO-8859-1"),"utf-8"); - forObject = forObject.replace("https://www.dmzj.com/js/ad/ad_12.js", ""); - forObject = forObject.replace("https://www.dmzj.com/js/ad/ad_13.js", ""); - forObject = forObject.replace("https://static.dmzj.com/ocomic/js/dmzjMhFinally-new.js", ""); - forObject = forObject.replace("https://static.dmzj.com/ocomic/js/dmzjMhFinally-new.js", ""); - forObject = forObject.replace("https://static.dmzj.com/module/js/float_code.js", ""); - forObject = forObject.replaceAll("", ""); - forObject = forObject.replaceAll("https://static.dmzj.com/public/js/dmzj-land-2015.6.js", ""); - forObject = forObject.replaceAll("", ""); - forObject = forObject.replaceAll("", ""); - forObject = forObject.replaceAll("", ""); - forObject = forObject.replaceAll("globalNav.js", ""); - forObject = forObject.replaceAll("TSB.js", ""); - forObject = forObject.replaceAll("https://images.dmzj.com/", "/manhua/images/"); - forObject = forObject.replaceAll("https://static.dmzj.com/", "/manhua/statics/"); - forObject = forObject + ""; - forObject = forObject + ""; -// forObject = forObject.replaceAll("/manhua/", "https://www.dmzj.com/") -// .replaceAll("笔趣岛", "酸味书屋") -// .replaceAll("笔趣阁", "酸味书屋") -// .replaceAll("class=\"mainNav independNav\"", "style='dispaly:none' class=\"mainNav independNav\"") -// .replaceAll("", "") -// .replaceFirst("", "") -// .replaceAll("", "") -// .replaceAll("https://zhannei.baidu.com/cse", "http://m.zinglizingli.xyz") -// .replaceAll("返回", "返回") -// .replaceAll("加入书架", "加入收藏") -// .replaceFirst("", "\n")//页面访问自动推送到百度 -// .replaceAll("", "")//去除广告 - ; - - -// if (requestURI.matches(SUANWEI_BOOK_HTML_REGEX)) { -// Pattern pattern = Pattern.compile("(.+)\\s+目录共\\d+章"); -// Matcher matcher = pattern.matcher(forObject); -// String title = ""; -// if (matcher.find()) { -// title = matcher.group(1); -// }//
  • 类别：武侠仙侠
    • -// pattern = Pattern.compile("

    作者：(.+)

    "); -// matcher = pattern.matcher(forObject); -// String author = ""; -// if (matcher.find()) { -// author = matcher.group(1); -// } -// pattern = Pattern.compile("(.+)"); -// matcher = pattern.matcher(forObject); -// String sort = ""; -// if (matcher.find()) { -// sort = matcher.group(1); -// } -// String desc = title + "，" + title + "小说最新章节免费在线阅读、最新章节列表，" + title + "小说最新更新免费提供，《" + title + "》是一本情节与文笔俱佳的" + sort + "小说，由作者" + author + "创建。"; -// -// forObject = forObject.replaceFirst("]+\"\\s*/?>", "");//[^>]+表示1个或多个不是>的字符 -// forObject = forObject.replaceFirst("", ""); -// -// -// } -// -// if ("/".equals(requestURI)) { -// forObject = forObject.replaceFirst("]+\"\\s*/?>", "");//[^>]+表示1个或多个不是>的字符 -// forObject = forObject.replaceFirst("", ""); -// -// -// } - } - } - - - } else { - - - Map oldParameterMap = req.getParameterMap(); - Map newParameterMap = new HashMap<>(); - Set> entries = oldParameterMap.entrySet(); - for (Map.Entry entry : entries) { - newParameterMap.put(entry.getKey(), entry.getValue()[0]); - } - - MultiValueMap map = new LinkedMultiValueMap<>(); - map.setAll(newParameterMap); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - HttpEntity> request = new HttpEntity<>(map, headers); - forObject = restTemplate.postForEntity("https://www.dmzj.com/" + requestURI.substring(8), request, String.class).getBody(); - // forObject = new String(forObject.getBytes("ISO-8859-1"),"utf-8"); - forObject = forObject.replaceAll("/manhua/", "https://www.dmzj.com/") - .replaceAll("笔趣岛", "酸味书屋") - .replaceAll("笔趣阁", "酸味书屋") - .replaceFirst("", "") - .replaceAll("书架", "笔记") - .replaceAll("返回", "返回") - ; - forObject = setBookURIToHTML(forObject, SUANWEI_BOOK_REGEX); - //resp.setCharacterEncoding("utf-8"); - //setContentType(postFix, resp); - - } - - - } catch (RuntimeException e) { - log.error(e.getMessage(), e); - if (e instanceof HttpClientErrorException && (((HttpClientErrorException) e).getStatusCode() == HttpStatus.NOT_FOUND)) { - //404 - resp.sendRedirect("/"); - return; - } else { - req.getRequestDispatcher("/mang.html").forward(servletRequest, servletResponse); - return; - } - - - //resp.setCharacterEncoding("utf-8"); - - } - - resp.getWriter().print(forObject); - return; - } - - private String addAttacDivForSearch(String forObject, String requestURI) { - try { - if (requestURI.endsWith(".html") || requestURI.equals("/")) { - String hotNewsDiv = cacheUtil.get("hotNewsDiv"); - if (hotNewsDiv == null) { - MultiValueMap mmap = new LinkedMultiValueMap<>(); - HttpHeaders headers = new HttpHeaders(); - headers.add("Host", "channel.chinanews.com"); - headers.add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"); - HttpEntity> request = new HttpEntity<>(mmap, headers); - String body = restTemplate.postForEntity("http://channel.chinanews.com/cns/cjs/sh.shtml", request, String.class).getBody(); - Pattern pattern = Pattern.compile("specialcnsdata\\s*=\\s*\\{\"docs\":(.+)};\\s+newslist\\s*=\\s*specialcnsdata;"); - Matcher matcher = pattern.matcher(body); - if (matcher.find()) { - String jsonResult = matcher.group(1); - if (jsonResult.length() > 5) { - List> list = new ObjectMapper().readValue(jsonResult, List.class); - StringBuilder hotContent = new StringBuilder(); - for (Map map : list) { - hotContent.append("
      \n"); - hotContent.append("
    • \n"); - hotContent.append(map.get("pubtime")); - hotContent.append("
      • \n"); - hotContent.append("
    • \n"); - hotContent.append(map.get("title")); - hotContent.append("
      • \n"); - hotContent.append("
    • \n"); - hotContent.append(map.get("content")); - hotContent.append("
      • \n"); - hotContent.append("
    • \n"); - hotContent.append(""); - hotContent.append("
      • \n"); - hotContent.append("
    \n"); - } - hotNewsDiv = "
    " + hotContent.toString() + "
    "; - cacheUtil.set("hotNewsDiv", hotNewsDiv, 60 * 60 * 24); - forObject = forObject.replaceFirst("", hotNewsDiv + ""); - } - } - } else { - forObject = forObject.replaceFirst("", hotNewsDiv + ""); - - } - } - } catch (Exception e) { - log.error(e.getMessage(), e); - } finally { - - } - - return forObject; - } - - private String setBookURIToHTML(String forObject, String regex) { - String result = forObject; - - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(forObject); - boolean isFind = matcher.find(); - if (isFind) { - - while (isFind) { - String booURI = matcher.group(1); - String htmlBooURI = booURI.substring(0, booURI.length()) + ".html"; - result = result.replaceFirst(booURI + "/", htmlBooURI); - isFind = matcher.find(); - } - - - } - - return result; - } - - private String postBiquta(HttpServletRequest req, String realUrl, Map otherParam) { - String forObject; - Map oldParameterMap = req.getParameterMap(); - Map newParameterMap = new HashMap<>(); - Set> entries = oldParameterMap.entrySet(); - for (Map.Entry entry : entries) { - newParameterMap.put(entry.getKey(), entry.getValue()[0]); - } - if (otherParam != null) { - newParameterMap.putAll(otherParam); - } - - MultiValueMap map = new LinkedMultiValueMap<>(); - map.setAll(newParameterMap); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - HttpEntity> request = new HttpEntity<>(map, headers); - forObject = restTemplate.postForEntity(realUrl, request, String.class).getBody(); - forObject = forObject.replaceAll("https://m.biquta.com", "http://m.zinglizingli.xyz") - .replaceAll("笔趣阁", "看小说吧") - .replaceAll("笔趣塔", "看小说吧") - .replaceFirst("看小说吧手机版-看小说吧", "看小说吧") - .replaceFirst("content=\"看小说吧\"", "content=\"小说阅读,小说排行,好看小说排行,热门小说排行,小说阅读手机版\"") - .replaceFirst("", "") - .replaceAll("我的书架", "轻小说精品小说") - .replaceFirst("阅读记录", "客户端下载") - .replaceFirst("", "\n") - ; - - forObject = setBookURIToHTML(forObject, XIYANGYANG_BOOK_REGEX); - return forObject; - } - - private void setContentType(String fileFix, HttpServletResponse resp) { - String contentType = "text/html"; - switch (fileFix) { - case "js": { - contentType = "application/javascript"; - break; - } - case "css": { - contentType = "text/css"; - break; - } - case "html": { - contentType = "text/html"; - break; - } - default: { - break; - } - } - resp.setContentType(contentType); - - - } - - @Override - public void destroy() { - - } - - -} diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java index e19583e..4f64c01 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java @@ -1,8 +1,8 @@ package xyz.zinglizingli.common.schedule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.Charsets; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -16,7 +16,6 @@ import xyz.zinglizingli.books.service.BookService; import xyz.zinglizingli.books.util.ExcutorUtils; import xyz.zinglizingli.common.utils.RestTemplateUtil; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -24,17 +23,21 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static java.util.regex.Pattern.compile; + +/** + * 更新书籍章节内容定时任务 + * + * @author 11797 + */ @Service +@RequiredArgsConstructor +@Slf4j public class CrawlBooksSchedule { + private final BookService bookService; - private Logger log = LoggerFactory.getLogger(CrawlBooksSchedule.class); - - - @Autowired - private BookService bookService; - - RestTemplate utf8RestTemplate = RestTemplateUtil.getInstance("utf-8"); + private RestTemplate utf8RestTemplate = RestTemplateUtil.getInstance(Charsets.UTF_8); @Value("${books.lowestScore}") @@ -51,204 +54,187 @@ public class CrawlBooksSchedule { private String picSavePath; - private boolean isExcuting = false; + /** + * 10分钟抓取一次 + */ + @Scheduled(fixedRate = 1000 * 60 * 10) + public void crawBooks() { + log.debug("crawlBooksSchedule执行中。。。。。。。。。。。。"); - @Scheduled(fixedRate = 1000 * 60 * 60 * 3) - public void crawBquge11BooksAtDay() throws Exception { - if (!isExcuting) { - isExcuting = true; - log.debug("crawlBooksSchedule执行中。。。。。。。。。。。。"); - - while (true) { - - try { - switch (websiteType) { - case 1: { - updateBiqudaoBooks(0); - break; - } - case 2: { - updateBiquTaBooks(0); - break; - } - } - Thread.sleep(1000 * 60 * 5); - } catch (Exception e) { - e.printStackTrace(); - } + switch (websiteType) { + case 1: { + updateBiqudaoBooks(0); + break; + } + case 2: { + updateBiquTaBooks(0); + break; + } + default: { + break; } - - } } - private void updateBiquTaBooks(int finalI) { + /** + * 从笔趣塔更新 + */ + private void updateBiquTaBooks(int bookClass) { String baseUrl = "https://m.biquta.la"; String catBookListUrlBase = baseUrl + "/class/"; - int page = 1;//起始页码 - int totalPage = page; - String catBookListUrl = catBookListUrlBase + finalI + "/" + page + ".html"; - String forObject = getByHttpClient(catBookListUrl); + String catBookListUrl = catBookListUrlBase + bookClass + "/" + 1 + ".html"; + String forObject = getByRestTemplate(catBookListUrl); if (forObject != null) { - //匹配分页数 - Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\""); + Pattern pattern = compile("value=\"(\\d+)/(\\d+)\""); Matcher matcher = pattern.matcher(forObject); boolean isFind = matcher.find(); System.out.println("匹配分页数" + isFind); if (isFind) { - int currentPage = Integer.parseInt(matcher.group(1)); - totalPage = Integer.parseInt(matcher.group(2)); //解析第一页书籍的数据 - Pattern bookPatten = Pattern.compile("href=\"/(\\d+_\\d+)/\""); - parseBiquTaBook(bookPatten, forObject, finalI, baseUrl, true); + Pattern bookPatten = compile("href=\"/(\\d+_\\d+)/\""); + parseBiquTaBook(bookPatten, forObject, bookClass, baseUrl); } } } - private void parseBiquTaBook(Pattern bookPatten, String forObject, int catNum, String baseUrl, boolean isUpdate) { - Matcher matcher2 = bookPatten.matcher(forObject); - boolean isFind = matcher2.find(); - Pattern scorePatten = Pattern.compile("(\\d+\\.\\d+)分"); + /** + * 解析笔趣塔数据 + */ + private void parseBiquTaBook(Pattern bookPatten, String forObject, int catNum, String baseUrl) { + Matcher bookMatcher = bookPatten.matcher(forObject); + + boolean isFind = bookMatcher.find(); + Pattern scorePatten = compile("(\\d+\\.\\d+)分"); Matcher scoreMatch = scorePatten.matcher(forObject); boolean scoreFind = scoreMatch.find(); - Pattern bookNamePatten = Pattern.compile("

    ([^/]+)

    "); + Pattern bookNamePatten = compile("

    ([^/]+)

    "); Matcher bookNameMatch = bookNamePatten.matcher(forObject); boolean isBookNameMatch = bookNameMatch.find(); - - System.out.println("匹配书籍url" + isFind); - - System.out.println("匹配分数" + scoreFind); - while (isFind && scoreFind && isBookNameMatch) { try { Float score = Float.parseFloat(scoreMatch.group(1)); - if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说 + if (score < lowestScore) { continue; } - String bokNum = matcher2.group(1); + String bokNum = bookMatcher.group(1); String bookUrl = baseUrl + "/" + bokNum + "/"; - String body = getByHttpClient(bookUrl); + String body = getByRestTemplate(bookUrl); if (body != null) { String bookName = bookNameMatch.group(1); - Pattern authorPatten = Pattern.compile(">作者：([^/]+)<"); + Pattern authorPatten = compile(">作者：([^/]+)<"); Matcher authoreMatch = authorPatten.matcher(body); if (authoreMatch.find()) { String author = authoreMatch.group(1); - Pattern statusPatten = Pattern.compile("状态：([^/]+)"); + Pattern statusPatten = compile("状态：([^/]+)"); Matcher statusMatch = statusPatten.matcher(body); if (statusMatch.find()) { String status = statusMatch.group(1); - Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)"); - Matcher updateTimeMatch = updateTimePatten.matcher(body); - if (updateTimeMatch.find()) { - String updateTimeStr = updateTimeMatch.group(1); - SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss"); - Date updateTime = format.parse(updateTimeStr); - Pattern picPatten = Pattern.compile("]+)\"\\s+onerror=\"this.src="); - Matcher picMather = picPatten.matcher(body); - if (picMather.find()) { - String picSrc = picMather.group(1); + Pattern updateTimePatten = compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)"); + Matcher updateTimeMatch = updateTimePatten.matcher(body); + if (updateTimeMatch.find()) { + String updateTimeStr = updateTimeMatch.group(1); + SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss"); + Date updateTime = format.parse(updateTimeStr); + Pattern picPatten = compile("]+)\"\\s+onerror=\"this.src="); + Matcher picMather = picPatten.matcher(body); + if (picMather.find()) { + String picSrc = picMather.group(1); - - Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)

    "); - Matcher descMatch = descPatten.matcher(body); - if (descMatch.find()) { - String desc = descMatch.group(1); + Pattern descPatten = compile("class=\"review\">([^<]+)

    "); + Matcher descMatch = descPatten.matcher(body); + if (descMatch.find()) { + String desc = descMatch.group(1); - Book book = new Book(); - book.setAuthor(author); - book.setCatid(catNum); - book.setBookDesc(desc); - book.setBookName(bookName); - book.setScore(score > 10 ? 8.0f : score); - book.setPicUrl(picSrc); - book.setBookStatus(status); - book.setUpdateTime(updateTime); + Book book = new Book(); + book.setAuthor(author); + book.setCatid(catNum); + book.setBookDesc(desc); + book.setBookName(bookName); + book.setScore(score > 10 ? 8.0f : score); + book.setPicUrl(picSrc); + book.setBookStatus(status); + book.setUpdateTime(updateTime); - List indexList = new ArrayList<>(); - List contentList = new ArrayList<>(); + List indexList = new ArrayList<>(); + List contentList = new ArrayList<>(); - //读取目录 - Pattern indexPatten = Pattern.compile("查看完整目录"); - Matcher indexMatch = indexPatten.matcher(body); - if (indexMatch.find()) { - String indexUrl = baseUrl + indexMatch.group(1); - String body2 = getByHttpClient(indexUrl); - if (body2 != null) { - Pattern indexListPatten = Pattern.compile("([^/]+)"); - Matcher indexListMatch = indexListPatten.matcher(body2); + //读取目录 + Pattern indexPatten = compile("查看完整目录"); + Matcher indexMatch = indexPatten.matcher(body); + if (indexMatch.find()) { + String indexUrl = baseUrl + indexMatch.group(1); + String body2 = getByRestTemplate(indexUrl); + if (body2 != null) { + Pattern indexListPatten = compile("([^/]+)"); + Matcher indexListMatch = indexListPatten.matcher(body2); - boolean isFindIndex = indexListMatch.find(); + boolean isFindIndex = indexListMatch.find(); - int indexNum = 0; + int indexNum = 0; - //查询该书籍已存在目录号 - List hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author); - //更新和插入分别开，插入只在凌晨做一次 - if ((isUpdate && hasIndexNum.size() > 0) || (!isUpdate && hasIndexNum.size() == 0)) { - while (isFindIndex) { - if (!hasIndexNum.contains(indexNum)) { + //查询该书籍已存在目录号 + List hasIndexNum = bookService.queryIndexCountByBookNameAndAuthor(bookName, author); + //更新和插入分别开，插入只在凌晨做一次 + if (hasIndexNum.size() > 0) { + while (isFindIndex) { + if (!hasIndexNum.contains(indexNum)) { - String contentUrl = baseUrl + indexListMatch.group(1); - String indexName = indexListMatch.group(2); + String contentUrl = baseUrl + indexListMatch.group(1); + String indexName = indexListMatch.group(2); - //查询章节内容 - String body3 = getByHttpClient(contentUrl); - if (body3 != null) { - Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读"); - String start = "『章节错误,点此举报』"; - String end = "『加入书签，方便阅读』"; - String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end)); - //TODO插入章节目录和章节内容 - BookIndex bookIndex = new BookIndex(); - bookIndex.setIndexName(indexName); - bookIndex.setIndexNum(indexNum); - indexList.add(bookIndex); - BookContent bookContent = new BookContent(); - bookContent.setContent(content); - bookContent.setIndexNum(indexNum); - contentList.add(bookContent); - - - } else { - break; - } + //查询章节内容 + String body3 = getByRestTemplate(contentUrl); + if (body3 != null) { + String start = "『章节错误,点此举报』"; + String end = "『加入书签，方便阅读』"; + String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end)); + //TODO插入章节目录和章节内容 + BookIndex bookIndex = new BookIndex(); + bookIndex.setIndexName(indexName); + bookIndex.setIndexNum(indexNum); + indexList.add(bookIndex); + BookContent bookContent = new BookContent(); + bookContent.setContent(content); + bookContent.setIndexNum(indexNum); + contentList.add(bookContent); + } else { + break; } - indexNum++; - isFindIndex = indexListMatch.find(); - } - if (indexList.size() == contentList.size() && indexList.size() > 0) { - ExcutorUtils.excuteFixedTask(new Runnable() { - @Override - public void run() { - bookService.saveBookAndIndexAndContent(book, indexList, contentList); - } - }); } + indexNum++; + isFindIndex = indexListMatch.find(); + } + + if (indexList.size() == contentList.size() && indexList.size() > 0) { + ExcutorUtils.excuteFixedTask(new Runnable() { + @Override + public void run() { + bookService.saveBookAndIndexAndContent(book, indexList, contentList); + } + }); + } } - - } @@ -256,6 +242,9 @@ public class CrawlBooksSchedule { } + + + } } } @@ -270,8 +259,8 @@ public class CrawlBooksSchedule { e.printStackTrace(); } finally { - matcher2.find(); - isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配 + bookMatcher.find(); + isFind = bookMatcher.find(); scoreFind = scoreMatch.find(); isBookNameMatch = bookNameMatch.find(); } @@ -280,27 +269,25 @@ public class CrawlBooksSchedule { } } - private void updateBiqudaoBooks(int finalI) { + /** + * 从笔趣岛更新 + */ + private void updateBiqudaoBooks(int bookClass) { String baseUrl = "https://m.biqudao.com"; String catBookListUrlBase = baseUrl + "/bqgeclass/"; - int page = 1;//起始页码 - int totalPage = page; - String catBookListUrl = catBookListUrlBase + finalI + "/" + page + ".html"; - String forObject = getByHttpClient(catBookListUrl); + int page = 1; + String catBookListUrl = catBookListUrlBase + bookClass + "/" + page + ".html"; + String forObject = getByRestTemplate(catBookListUrl); if (forObject != null) { - //匹配分页数 - Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\""); + Pattern pattern = compile("value=\"(\\d+)/(\\d+)\""); Matcher matcher = pattern.matcher(forObject); boolean isFind = matcher.find(); System.out.println("匹配分页数" + isFind); if (isFind) { - int currentPage = Integer.parseInt(matcher.group(1)); - totalPage = Integer.parseInt(matcher.group(2)); //解析第一页书籍的数据 - Pattern bookPatten = Pattern.compile("href=\"/(bqge\\d+)/\""); - //白天更新 - parseBiquDaoBook(bookPatten, forObject, finalI, baseUrl, true); + Pattern bookPatten = compile("href=\"/(bqge\\d+)/\""); + parseBiquDaoBook(bookPatten, forObject, baseUrl); } } @@ -308,160 +295,53 @@ public class CrawlBooksSchedule { } - //@Scheduled(cron = "0 0 2 * * ?")磁盘空间不足，暂时不抓新书 - //暂定2小说，只爬分类前3本书，一共3*7=21本书，爬等以后书籍多了之后，会适当缩短更新间隔 - public void crawBquge11BooksAtNight() throws Exception { - final String baseUrl = "https://m.biqudao.com"; - log.debug("crawlBooksSchedule执行中。。。。。。。。。。。。"); + /** + * 解析笔趣岛数据 + */ + private void parseBiquDaoBook(Pattern bookPatten, String forObject, String baseUrl) { - -//①爬分类列表的书籍url和总页数 -// https: -////m.biquta.la/class/1/1.html -// https: -////m.biquta.la/class/2/1.html -// https: -////m.biquta.la/class/2/2.html -// -// -// https: -////m.biquta.la/class/2/2.html -// -// -// -//
    -// -//
    -//

    苍穹九变

    -//

    作者：风起闲云

    -//
    -//
    7.5 分
    -//     -//
    -// -// - //第一周期全部书拉取完后，可进行第二周期，只拉取前面几页的数据，拉取时间间隔变小 - log.debug("crawlBooksSchedule循环执行中。。。。。。。。。。。。"); - //List classIdList = new ArrayList<>(Arrays.asList(new Integer[]{1,2,3,4,5,6,7})); - // for (int i = 1; i <= 7; i++) { - - // log.debug("crawlBooksSchedule分类"+i+"执行中。。。。。。。。。。。。"); - - // int finalI = i; - /* new Thread( - () -> {*/ - - try { - //先随机更新分类 - //Random random = new Random(); - //int finalI = classIdList.get(new Random().nextInt(classIdList.size())); - //classIdList.remove(finalI); - int finalI = 0; - //拼接分类URL - int page = 1;//起始页码 - int totalPage = page; - String catBookListUrl = baseUrl + "/bqgeclass/" + finalI + "/" + page + ".html"; - String forObject = getByHttpClient(catBookListUrl); - if (forObject != null) { - //匹配分页数 - Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\""); - Matcher matcher = pattern.matcher(forObject); - boolean isFind = matcher.find(); - System.out.println("匹配分页数" + isFind); - if (isFind) { - int currentPage = Integer.parseInt(matcher.group(1)); - totalPage = Integer.parseInt(matcher.group(2)); - //解析第一页书籍的数据 - Pattern bookPatten = Pattern.compile("href=\"/(bqge\\d+)/\""); - //晚上插入 - parseBiquDaoBook(bookPatten, forObject, finalI, baseUrl, false); - while (currentPage < totalPage) { - if (new Date().getHours() > 5) { - break; - } - catBookListUrl = baseUrl + "/bqgeclass/" + finalI + "/" + (currentPage + 1) + ".html"; - forObject = getByHttpClient(catBookListUrl); - if (forObject != null) { - //匹配分页数 - matcher = pattern.matcher(forObject); - isFind = matcher.find(); - - if (isFind) { - currentPage = Integer.parseInt(matcher.group(1)); - totalPage = Integer.parseInt(matcher.group(2)); - parseBiquDaoBook(bookPatten, forObject, finalI, baseUrl, false); - } - } else { - currentPage++; - } - } - } - } - } catch (Exception e) { - e.printStackTrace(); - } - - /* } - ).start();*/ - - - // } - - - } - - private void parseBiquDaoBook(Pattern bookPatten, String forObject, int catNum, String baseUrl, boolean isUpdate) { - - Matcher matcher2 = bookPatten.matcher(forObject); - boolean isFind = matcher2.find(); - Pattern scorePatten = Pattern.compile("(\\d+\\.\\d+)分"); + Matcher bookMatcher = bookPatten.matcher(forObject); + boolean isFind = bookMatcher.find(); + Pattern scorePatten = compile("(\\d+\\.\\d+)分"); Matcher scoreMatch = scorePatten.matcher(forObject); boolean scoreFind = scoreMatch.find(); - Pattern bookNamePatten = Pattern.compile("

    ([^/]+)

    "); + Pattern bookNamePatten = compile("

    ([^/]+)

    "); Matcher bookNameMatch = bookNamePatten.matcher(forObject); boolean isBookNameMatch = bookNameMatch.find(); - - System.out.println("匹配书籍url" + isFind); - - System.out.println("匹配分数" + scoreFind); - while (isFind && scoreFind && isBookNameMatch) { try { Float score = Float.parseFloat(scoreMatch.group(1)); - if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说 + if (score < lowestScore) { continue; } - String bokNum = matcher2.group(1); + String bokNum = bookMatcher.group(1); String bookUrl = baseUrl + "/" + bokNum + "/"; - String body = getByHttpClient(bookUrl); + String body = getByRestTemplate(bookUrl); if (body != null) { String bookName = bookNameMatch.group(1); - Pattern authorPatten = Pattern.compile("
  • 作者：([^/]+)
    • "); + Pattern authorPatten = compile("
  • 作者：([^/]+)
    • "); Matcher authoreMatch = authorPatten.matcher(body); if (authoreMatch.find()) { String author = authoreMatch.group(1); - Pattern statusPatten = Pattern.compile("状态：([^/]+)"); + Pattern statusPatten = compile("状态：([^/]+)"); Matcher statusMatch = statusPatten.matcher(body); if (statusMatch.find()) { String status = statusMatch.group(1); - Pattern catPatten = Pattern.compile("类别：([^/]+)"); + Pattern catPatten = compile("类别：([^/]+)"); Matcher catMatch = catPatten.matcher(body); if (catMatch.find()) { String catName = catMatch.group(1); + int catNum; switch (catName) { - case "玄幻奇幻": { - catNum = 1; - break; - } case "武侠仙侠": { catNum = 2; break; @@ -491,19 +371,19 @@ public class CrawlBooksSchedule { break; } } - Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)"); + Pattern updateTimePatten = compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)"); Matcher updateTimeMatch = updateTimePatten.matcher(body); if (updateTimeMatch.find()) { String updateTimeStr = updateTimeMatch.group(1); SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss"); Date updateTime = format.parse(updateTimeStr); - Pattern picPatten = Pattern.compile("]+)\"\\s+onerror=\"this.src="); + Pattern picPatten = compile("]+)\"\\s+onerror=\"this.src="); Matcher picMather = picPatten.matcher(body); if (picMather.find()) { String picSrc = picMather.group(1); - Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)

    "); + Pattern descPatten = compile("class=\"review\">([^<]+)

    "); Matcher descMatch = descPatten.matcher(body); if (descMatch.find()) { String desc = descMatch.group(1); @@ -523,13 +403,13 @@ public class CrawlBooksSchedule { List contentList = new ArrayList<>(); //读取目录 - Pattern indexPatten = Pattern.compile("查看完整目录"); + Pattern indexPatten = compile("查看完整目录"); Matcher indexMatch = indexPatten.matcher(body); if (indexMatch.find()) { String indexUrl = baseUrl + indexMatch.group(1); - String body2 = getByHttpClient(indexUrl); + String body2 = getByRestTemplate(indexUrl); if (body2 != null) { - Pattern indexListPatten = Pattern.compile("([^/]+)"); + Pattern indexListPatten = compile("([^/]+)"); Matcher indexListMatch = indexListPatten.matcher(body2); boolean isFindIndex = indexListMatch.find(); @@ -537,9 +417,9 @@ public class CrawlBooksSchedule { int indexNum = 0; //查询该书籍已存在目录号 - List hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author); + List hasIndexNum = bookService.queryIndexCountByBookNameAndAuthor(bookName, author); //更新和插入分别开，插入只在凌晨做一次 - if ((isUpdate && hasIndexNum.size() > 0) || (!isUpdate && hasIndexNum.size() == 0)) { + if (hasIndexNum.size() > 0) { while (isFindIndex) { if (!hasIndexNum.contains(indexNum)) { @@ -548,9 +428,8 @@ public class CrawlBooksSchedule { //查询章节内容 - String body3 = getByHttpClient(contentUrl); + String body3 = getByRestTemplate(contentUrl); if (body3 != null) { - Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读"); String start = "『章节错误,点此举报』"; String end = "『加入书签，方便阅读』"; String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end)); @@ -576,12 +455,7 @@ public class CrawlBooksSchedule { } if (indexList.size() == contentList.size() && indexList.size() > 0) { - ExcutorUtils.excuteFixedTask(new Runnable() { - @Override - public void run() { - bookService.saveBookAndIndexAndContent(book, indexList, contentList); - } - }); + ExcutorUtils.excuteFixedTask(() -> bookService.saveBookAndIndexAndContent(book, indexList, contentList)); } } @@ -610,8 +484,8 @@ public class CrawlBooksSchedule { e.printStackTrace(); } finally { - matcher2.find(); - isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配 + bookMatcher.find(); + isFind = bookMatcher.find(); scoreFind = scoreMatch.find(); isBookNameMatch = bookNameMatch.find(); } @@ -621,19 +495,8 @@ public class CrawlBooksSchedule { } - private String getByHttpClient(String url) { + private String getByRestTemplate(String url) { try { - /* HttpClient httpClient = new DefaultHttpClient(); - HttpGet getReq = new HttpGet(catBookListUrl); - getReq.setHeader("user-agent", "Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1"); - HttpResponse execute = httpClient.execute(getReq); - if (execute.getStatusLine().getStatusCode() == HttpStatus.OK.value()) { - HttpEntity entity = execute.getEntity(); - return EntityUtils.toString(entity, "utf-8"); - } else { - return null; - }*/ - //经测试restTemplate比httpClient效率高出很多倍，所有选择restTemplate ResponseEntity forEntity = utf8RestTemplate.getForEntity(url, String.class); if (forEntity.getStatusCode() == HttpStatus.OK) { return forEntity.getBody(); @@ -641,173 +504,10 @@ public class CrawlBooksSchedule { return null; } } catch (Exception e) { - e.printStackTrace(); + log.error(e.getMessage(), e); return null; } } - /*** - * 解析书籍详情之后的页面 - */ - private void parseBook(Pattern bookPatten, String forObject, RestTemplate restTemplate, int catNum, String baseUrl) throws ParseException { - Matcher matcher2 = bookPatten.matcher(forObject); - boolean isFind = matcher2.find(); - Pattern scorePatten = Pattern.compile("(\\d+\\.\\d+)分"); - Matcher scoreMatch = scorePatten.matcher(forObject); - boolean scoreFind = scoreMatch.find(); - Pattern bookNamePatten = Pattern.compile("

    ([^/]+)

    "); - Matcher bookNameMatch = bookNamePatten.matcher(forObject); - boolean isBookNameMatch = bookNameMatch.find(); - - Pattern authorPatten = Pattern.compile(">作者：([^/]+)<"); - Matcher authoreMatch = authorPatten.matcher(forObject); - boolean isFindAuthor = authoreMatch.find(); - - - System.out.println("匹配书籍url" + isFind); - - System.out.println("匹配分数" + scoreFind); - while (isFind && scoreFind && isBookNameMatch && isFindAuthor) { - - try { - Float score = Float.parseFloat(scoreMatch.group(1)); - - if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说 - continue; - } - String bookName = bookNameMatch.group(1); - String author = authoreMatch.group(1); - - String bokNum = matcher2.group(1); - String bookUrl = baseUrl + "/" + bokNum + "/"; - - ResponseEntity forEntity = restTemplate.getForEntity(bookUrl, String.class); - if (forEntity.getStatusCode() == HttpStatus.OK) { - - String body = forEntity.getBody(); - - Pattern statusPatten = Pattern.compile("状态：([^/]+)"); - Matcher statusMatch = statusPatten.matcher(body); - if (statusMatch.find()) { - String status = statusMatch.group(1); - Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)"); - Matcher updateTimeMatch = updateTimePatten.matcher(body); - if (updateTimeMatch.find()) { - String updateTimeStr = updateTimeMatch.group(1); - SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss"); - Date updateTime = format.parse(updateTimeStr); - Pattern picPatten = Pattern.compile("]+)\"\\s+onerror=\"this.src="); - Matcher picMather = picPatten.matcher(body); - if (picMather.find()) { - String picSrc = picMather.group(1); - - Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)

    "); - Matcher descMatch = descPatten.matcher(body); - if (descMatch.find()) { - String desc = descMatch.group(1); - - - Book book = new Book(); - book.setAuthor(author); - book.setCatid(catNum); - book.setBookDesc(desc); - book.setBookName(bookName); - book.setScore(score > 10 ? 8.0f : score); - book.setPicUrl(picSrc); - book.setBookStatus(status); - book.setUpdateTime(updateTime); - - List indexList = new ArrayList<>(); - List contentList = new ArrayList<>(); - - //读取目录 - Pattern indexPatten = Pattern.compile("查看完整目录"); - Matcher indexMatch = indexPatten.matcher(body); - if (indexMatch.find()) { - String indexUrl = baseUrl + indexMatch.group(1); - ResponseEntity forEntity1 = restTemplate.getForEntity(indexUrl, String.class); - if (forEntity1.getStatusCode() == HttpStatus.OK) { - String body2 = forEntity1.getBody(); - Pattern indexListPatten = Pattern.compile("([^/]+)"); - Matcher indexListMatch = indexListPatten.matcher(body2); - - boolean isFindIndex = indexListMatch.find(); - - int indexNum = 0; - - //查询该书籍已存在目录号 - List hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author); - - while (isFindIndex) { - if (!hasIndexNum.contains(indexNum)) { - - String contentUrl = baseUrl + indexListMatch.group(1); - String indexName = indexListMatch.group(2); - - - //查询章节内容 - ResponseEntity forEntity2 = restTemplate.getForEntity(contentUrl, String.class); - if (forEntity2.getStatusCode() == HttpStatus.OK) { - String body3 = forEntity2.getBody(); - Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读"); - String start = "『章节错误,点此举报』"; - String end = "『加入书签，方便阅读』"; - String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end)); - //TODO插入章节目录和章节内容 - BookIndex bookIndex = new BookIndex(); - bookIndex.setIndexName(indexName); - bookIndex.setIndexNum(indexNum); - indexList.add(bookIndex); - BookContent bookContent = new BookContent(); - bookContent.setContent(content); - bookContent.setIndexNum(indexNum); - contentList.add(bookContent); - - - } else { - break; - } - - } - - indexNum++; - isFindIndex = indexListMatch.find(); - } - if (indexList.size() == contentList.size() && indexList.size() > 0) { - bookService.saveBookAndIndexAndContent(book, indexList, contentList); - } - - - } - } - - - } - - - } - } - - - } - - } - - } catch (Exception e) { - - e.printStackTrace(); - - } finally { - matcher2.find(); - isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配 - scoreFind = scoreMatch.find(); - isBookNameMatch = bookNameMatch.find(); - isFindAuthor = authoreMatch.find(); - } - - - } - - } } diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendEmaillSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendEmaillSchedule.java deleted file mode 100644 index bda344f..0000000 --- a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendEmaillSchedule.java +++ /dev/null @@ -1,56 +0,0 @@ -package xyz.zinglizingli.common.schedule; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import xyz.zinglizingli.books.constant.CacheKeyConstans; -import xyz.zinglizingli.books.service.MailService; -import xyz.zinglizingli.books.util.RandomValueUtil; -import xyz.zinglizingli.common.cache.CommonCacheUtil; - -import java.util.Random; - - -/* -主动推送：最为快速的提交方式， -建议您将站点当天新产出链接立即通过此方式推送给百度， -以保证新链接可以及时被百度收录。 -*/ -@Service -public class SendEmaillSchedule { - - @Autowired - private CommonCacheUtil cacheUtil; - - @Autowired - private MailService mailService; - - - private Logger log = LoggerFactory.getLogger(SendEmaillSchedule.class); - - - // @Scheduled(fixedRate = 1000*60*60*24) - public void sendEmaill() { - System.out.println("SendEmaillSchedule。。。。。。。。。。。。。。。"); - - for(int i = 0 ; i < 1000; i++){ - String email = RandomValueUtil.getEmail(); - if(cacheUtil.get(CacheKeyConstans.EMAIL_URL_PREFIX_KEY+email)!=null){ - continue; - } - cacheUtil.setObject(CacheKeyConstans.EMAIL_URL_PREFIX_KEY+email,email,60*60*24*30); - String subject = "推荐一个看小说的弹幕网站"; - String content = "精品小说楼是国内优秀的小说弹幕网站,精品小说楼提供海量热门网络小说,日本轻小说,国产轻小说,动漫小说,轻小说在线阅读和TXT小说下载,致力于网络精品小说的收集,智能计算小说评分,打造小说精品排行榜,致力于无广告无弹窗的小说阅读环境。" + - "
    点击进入" - +"
    "; - mailService.sendHtmlMail(email, subject, content); - try { - Thread.sleep(new Random().nextInt(1000*60*10)+1000*60); - } catch (InterruptedException e) { - log.error(e.getMessage(),e); - } - } - - } -} diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendUrlSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendUrlSchedule.java deleted file mode 100644 index 8175fed..0000000 --- a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendUrlSchedule.java +++ /dev/null @@ -1,84 +0,0 @@ -package xyz.zinglizingli.common.schedule; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.client.RestTemplate; -import xyz.zinglizingli.books.service.BookService; -import xyz.zinglizingli.common.cache.CommonCacheUtil; -import xyz.zinglizingli.common.utils.RestTemplateUtil; - -import java.util.*; - - -/* -主动推送：最为快速的提交方式， -建议您将站点当天新产出链接立即通过此方式推送给百度， -以保证新链接可以及时被百度收录。 -*/ -@Service -public class SendUrlSchedule { - - @Autowired - private CommonCacheUtil cacheUtil; - - @Autowired - private BookService bookService; - - @Value("${baidu.record.ids}") - private String recordedIds; - - private Logger log = LoggerFactory.getLogger(SendUrlSchedule.class); - - - //@Scheduled(cron = "0 0 1 * * 1") - public void sendAllBookToBaidu() { - System.out.println("sendAllBookToBaidu。。。。。。。。。。。。。。。"); - - List recordedIdsList = Arrays.asList(recordedIds.split(",")); - List idList = bookService.queryEndBookIdList(); - RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8"); - - - String reqBody = ""; - for (String id : idList) { - try { - if (!recordedIdsList.contains(id)) { - reqBody += ("https://www.zinglizingli.xyz/book/" + id + ".html" + "\n"); - //reqBody+=("http://www.zinglizingli.xyz/book/"+id+".html"+"\n"); - if (reqBody.length() > 2000) { - MultiValueMap map = new LinkedMultiValueMap<>(); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.TEXT_PLAIN); - //headers.add("User-Agent","curl/7.12.1"); - headers.add("Host", "data.zz.baidu.com"); - headers.setContentLength(reqBody.length()); - HttpEntity request = new HttpEntity<>(reqBody, headers); - System.out.println("推送数据：" + reqBody); - ResponseEntity stringResponseEntity = restTemplate.postForEntity("http://data.zz.baidu.com/urls?site=www.zinglizingli.xyz&token=IuK7oVrPKe3U606x", request, String.class); - System.out.println("推送URL结果：code:" + stringResponseEntity.getStatusCode().value() + ",body:" + stringResponseEntity.getBody()); - Thread.sleep(1000 * 10); - System.out.println("推送数据：" + reqBody); - stringResponseEntity = restTemplate.postForEntity("http://data.zz.baidu.com/urls?appid=1643715155923937&token=fkEcTlId6Cf21Sz3&type=batch", request, String.class); - System.out.println("推送URL结果：code:" + stringResponseEntity.getStatusCode().value() + ",body:" + stringResponseEntity.getBody()); - - reqBody = ""; - Thread.sleep(1000 * 10); - } - } - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } - - } -} diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendWeiboSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendWeiboSchedule.java deleted file mode 100644 index e689273..0000000 --- a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/SendWeiboSchedule.java +++ /dev/null @@ -1,497 +0,0 @@ -package xyz.zinglizingli.common.schedule; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.client.RestTemplate; -import xyz.zinglizingli.books.po.Book; -import xyz.zinglizingli.books.service.BookService; -import xyz.zinglizingli.common.cache.CommonCacheUtil; -import xyz.zinglizingli.common.utils.RestTemplateUtil; - -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@Service -public class SendWeiboSchedule { - - @Autowired - private CommonCacheUtil cacheUtil; - - @Autowired - private BookService bookService; - - private Logger log = LoggerFactory.getLogger(SendWeiboSchedule.class); - - private boolean isExcuting = false;//是否正在执行 - - private long excuteNum = 0; - - @Value("${search.schedule.isRunExcute}") - private String isRunExcute;//是否在运行时就执行sendAtNight定时器 - - @Value("${browser.cookie}") - private String cookieStr; - - private static final String BOOKNAME_CACHE_PREFIX = "bookName_Cache_Prefix:"; - - - //@Scheduled(fixedRate = 1000 * 60 * 35) - public void sendAtDay() { - log.debug("sendWeoboSchedule执行中。。。。。。。。。。。。"); - if (!isExcuting) { - isExcuting = true; - excuteNum++; - //long sleepMillis = 1000 * 60 * 5; - long sleepMillis = 1000 * 60 * 5; - try { - - String name; - String desc; - String author; - String bookNum; - String resultCode; - long realSleepMillis; - - RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8"); - - //发送数据库中的一篇文章 - Map dataMap = bookService.queryNewstBook(); - log.debug("dataMap大小：" + dataMap.size()); - if (dataMap.size() > 1) { - Book book = (Book) dataMap.get("book"); - String newstIndexName = (String) dataMap.get("newstIndexName"); - name = newstIndexName + "_" + book.getBookName(); - desc = book.getBookDesc(); - author = book.getAuthor(); - bookNum = new Random().nextInt(100) + ""; - realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000; - log.debug("发送微博书籍名：" + book.getBookName()); - if (!name.equals(cacheUtil.get(BOOKNAME_CACHE_PREFIX + name))) { - resultCode = sendOneSiteWeibo(restTemplate, book.getBookName(), newstIndexName, author, desc, "精品小说楼", bookNum, "https://www.zinglizingli.xyz/book/" + book.getId() + ".html"); - log.debug("发送微博书籍名：" + book.getBookName() + " 状态码：" + resultCode); - if ("{\"code\":\"A00006\"}".equals(resultCode)) { - cacheUtil.set(BOOKNAME_CACHE_PREFIX + name, name, 60 * 60 * 24 * 30); - } - Thread.sleep(realSleepMillis + 32); - } - } - - - //分享喜羊羊小说网 - String url2 = "http://m.zinglizingli.xyz/class/0/1.html"; - - ResponseEntity forEntity2 = restTemplate.getForEntity(url2, String.class); - - - String forObject2 = forEntity2.getBody(); - Pattern pattern = Pattern.compile("
    " + - "\\s*" + - "\\s*
    " + - "\\s*

    (.*)

    " + - "\\s*

    (.*)

    " + - "\\s*
    " + - "\\s*
    (\\d*\\.\\d*)分
    " + - "\\s*    " + - "\\s*
    " + - "\\s*

    ((.*))

    \\s*" + - ""); - Matcher match = pattern.matcher(forObject2); - boolean isFind = match.find(); - if (isFind) { - while (isFind) { - - float score = Float.parseFloat(match.group(4)); - - if (score >= 7.0) { - - bookNum = match.group(1); - String href = "http://m.zinglizingli.xyz/" + bookNum + ".html"; - name = match.group(2); - if (!name.equals(cacheUtil.get(BOOKNAME_CACHE_PREFIX + name))) { - author = match.group(3); - desc = match.group(5); - - resultCode = sendOneSiteWeibo(restTemplate, name, "", author, desc, "看小说吧", bookNum, href); - if ("{\"code\":\"A00006\"}".equals(resultCode)) { - cacheUtil.set(BOOKNAME_CACHE_PREFIX + name, name, 60 * 60 * 24 * 30); - } - - realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000; - Thread.sleep(realSleepMillis); - - } - } - isFind = match.find(); - } - - - } - - } catch (Exception e) { - log.error(e.getMessage(), e); - - } finally { - isExcuting = false; - } - - } - - } - - //@Scheduled(fixedRate = 1000 * 60 * 35) - //@Scheduled(fixedRate = 1000 * 60 * 5) - /*public void sendAtDay() { - if (!isExcuting) { - isExcuting = true; - excuteNum++; - //long sleepMillis = 1000 * 60 * 5; - long sleepMillis = 1000 * 60 * 25; - try { - RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8"); - //分享酸味书屋 - String url = "http://www.zinglizingli.xyz/paihangbang_lastupdate/1.html"; - - //分享喜羊羊小说网 - String url2 = "http://m.zinglizingli.xyz/class/0/1.html"; - - ResponseEntity forEntity = restTemplate.getForEntity(url, String.class); - ResponseEntity forEntity2 = restTemplate.getForEntity(url2, String.class); - - String forObject = forEntity.getBody(); - Pattern pattern = Pattern.compile("
  • \n" + - "(.*)\n" + - "(.*)\n" + - "(.*)\\.\\.\\.\n" + - "(\\d*)人在看\n" + - "
    • "); - Matcher match = pattern.matcher(forObject); - boolean isFind = match.find(); - if (isFind) { - while (isFind) { - - int lookNum = Integer.parseInt(match.group(5)); - if (lookNum > 5000) { - String bookNum = match.group(1); - String href = "http://www.zinglizingli.xyz/" + bookNum + ".html"; - String name = match.group(2); - log.debug(excuteNum + ":" + name + "_BOOKNAME_CACHE:" + cacheUtil.get(BOOKNAME_CACHE_PREFIX + name)); - if (!name.equals(cacheUtil.get(BOOKNAME_CACHE_PREFIX + name))) { - - String author = match.group(3); - String desc = match.group(4); - log.debug(excuteNum + ":" + name); - String resultCode = sendOneSiteWeibo(restTemplate, name, author, desc, "酸味书屋", bookNum, href); - log.debug(excuteNum + ":" + name + ":" + resultCode); - log.debug(excuteNum + ":resultCode=={\"code\":\"A00006\"}" + "{\"code\":\"A00006\"}".equals(resultCode)); - - if ("{\"code\":\"A00006\"}".equals(resultCode)) { - cacheUtil.set(BOOKNAME_CACHE_PREFIX + name, name, 60 * 60 * 24 * 30); - } - long realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000; - Thread.sleep(realSleepMillis); - } - - } - isFind = match.find(); - } - - - String forObject2 = forEntity2.getBody(); - pattern = Pattern.compile("
    " + - "\\s*" + - "\\s*
    " + - "\\s*

    (.*)

    " + - "\\s*

    (.*)

    " + - "\\s*
    " + - "\\s*
    (\\d*\\.\\d*)分
    " + - "\\s*    " + - "\\s*
    " + - "\\s*

    ((.*))

    \\s*" + - ""); - match = pattern.matcher(forObject2); - isFind = match.find(); - if (isFind) { - while (isFind) { - - float score = Float.parseFloat(match.group(4)); - - if (score >= 7.0) { - - String bookNum = match.group(1); - String href = "http://m.zinglizingli.xyz/" + bookNum + ".html"; - String name = match.group(2); - if (!name.equals(cacheUtil.get(BOOKNAME_CACHE_PREFIX + name))) { - String author = match.group(3); - String desc = match.group(5); - - String resultCode = sendOneSiteWeibo(restTemplate, name, author, desc, "喜羊羊小说网", bookNum, href); - if ("{\"code\":\"A00006\"}".equals(resultCode)) { - cacheUtil.set(BOOKNAME_CACHE_PREFIX + name, name, 60 * 60 * 24 * 30); - } - - long realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000; - Thread.sleep(realSleepMillis); - - } - } - isFind = match.find(); - } - - - } - } - - } catch (Exception e) { - log.error(e.getMessage(),e); - - } finally { - isExcuting = false; - } - - } - - } -*/ - //19点到23点，1点到4点每隔50分钟执行一次,20本书*2分钟+空闲时间 - //@Scheduled(cron = "0 */50 19-23,1-4 * * ?") - - /* public void sendAtNight() throws InterruptedException, IOException { - if (!isExcuting) { - isExcuting = true; - log.info("sendAtNight定时器开始执行。。。。"); - long sleepMillis = 1000 * 60 * 2; - sendAllSiteWeibo(sleepMillis); - Thread.sleep(1000 * 60 * 10); - isExcuting = false; - } - - - }*/ - - - //6点到17点每隔1小时执行一次，20本书*5分钟+空闲时间 - //@Scheduled(cron = "0 0 6-17/1 * * ?") - /* public void sendAtDayTime() throws InterruptedException, IOException { - - if (!isExcuting) { - isExcuting = true; - log.info("sendAtDayTime定时器开始执行。。。。"); - long sleepMillis = 1000 * 60 * 5; - sendAllSiteWeibo(sleepMillis); - Thread.sleep(1000 * 60 * 10); - isExcuting = false; - } - - }*/ - - /*private void sendAllSiteWeibo(long sleepMillis) throws InterruptedException { - RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8"); - //分享酸味书屋 - String url = "http://www.zinglizingli.xyz/paihangbang_lastupdate/1.html"; - - //分享喜羊羊小说网 - String url2 = "http://m.zinglizingli.xyz/class/0/1.html"; - - ResponseEntity forEntity = restTemplate.getForEntity(url, String.class); - ResponseEntity forEntity2 = restTemplate.getForEntity(url2, String.class); - - String forObject = forEntity.getBody(); - Pattern pattern = Pattern.compile("
  • \n" + - "(.*)\n" + - "(.*)\n" + - "(.*)\\.\\.\\.\n" + - "(\\d*)人在看\n" + - "
    • "); - Matcher match = pattern.matcher(forObject); - boolean isFind = match.find(); - if (isFind) { - while (isFind) { - - int lookNum = Integer.parseInt(match.group(5)); - if (lookNum > 5000) { - String bookNum = match.group(1); - String href = "http://www.zinglizingli.xyz/" + bookNum; - String name = match.group(2); - String author = match.group(3); - String desc = match.group(4); - sendOneSiteWeibo(restTemplate, name, author, desc, "酸味书屋", bookNum, href); - long realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000; - Thread.sleep(realSleepMillis); - } - - isFind = match.find(); - - } - } - - - String forObject2 = forEntity2.getBody(); - pattern = Pattern.compile("
    " + - "\\s*" + - "\\s*
    " + - "\\s*

    (.*)

    " + - "\\s*

    (.*)

    " + - "\\s*
    " + - "\\s*
    (\\d*\\.\\d*)分
    " + - "\\s*    " + - "\\s*
    " + - "\\s*

    ((.*))

    \\s*" + - ""); - match = pattern.matcher(forObject2); - isFind = match.find(); - if (isFind) { - while (isFind) { - - float score = Float.parseFloat(match.group(4)); - - if (score >= 7.0) { - - String bookNum = match.group(1); - String href = "http://m.zinglizingli.xyz/" + bookNum; - String name = match.group(2); - String author = match.group(3); - String desc = match.group(5); - - sendOneSiteWeibo(restTemplate, name, author, desc, "喜羊羊小说网", bookNum, href); - - long realSleepMillis = sleepMillis + (new Random().nextInt(15)) * 60 * 1000; - Thread.sleep(realSleepMillis); - } - - isFind = match.find(); - - - } - } - }*/ - - - public static void main(String[] args) throws Exception { - RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8"); - //分享酸味书屋 - String url = "http://www.zinglizingli.xyz/paihangbang_lastupdate/1.html"; - ResponseEntity forEntity = restTemplate.getForEntity(url, String.class); - String forObject = forEntity.getBody(); - Pattern pattern = Pattern.compile("
  • \n" + - "(.*)\n" + - "(.*)\n" + - "(.*)\\.\\.\\.\n" + - "\\d*人在看\n" + - "
    • "); - Matcher match = pattern.matcher(forObject); - boolean isFind = match.find(); - if (isFind) { - while (isFind) { - String bookNum = match.group(1); - String href = "http://www.zinglizingli.xyz/" + bookNum; - String name = match.group(2); - String author = match.group(3); - String desc = match.group(4); - - - isFind = match.find(); - - } - } - - //分享喜羊羊小说网 - url = "http://m.zinglizingli.xyz/class/0/1.html"; - forEntity = restTemplate.getForEntity(url, String.class); - forObject = forEntity.getBody(); - pattern = Pattern.compile("
    " + - "\\s*" + - "\\s*
    " + - "\\s*

    (.*)

    " + - "\\s*

    (.*)

    " + - "\\s*
    " + - "\\s*
    \\d*\\.\\d*分
    " + - "\\s*    " + - "\\s*
    " + - "\\s*

    ((.*))

    \\s*" + - ""); - match = pattern.matcher(forObject); - isFind = match.find(); - if (isFind) { - while (isFind) { - String bookNum = match.group(1); - String href = "http://m.zinglizingli.xyz/" + bookNum; - String name = match.group(2); - String author = match.group(3); - String desc = match.group(4); - // sendOneSiteWeibo(restTemplate, name, author, desc, "喜羊羊小说网", bookNum, href); - - - isFind = match.find(); - - - } - } - - - } - - private String sendOneSiteWeibo(RestTemplate template, String bookName, String indexName, String author, String desc, String wapName, String bookNum, String href) { - String baseUrl = "http://service.weibo.com/share/aj_share.php"; - Map param = new HashMap<>(); - /*String content = bookName + "小说最新章节列表," + bookName + "小说免费在线阅读," + bookName + - "小说TXT下载,尽在" + wapName +href+ "\n"; - if(indexName != null){ - content+=("最新章节："+indexName+"\n"); - } - content = content + "作者："+(author.replace("作者：","")) + "\n"; - content += ("简介："+desc.replace("简介：",""));*/ - String content = bookName+"最新章节,小说"+bookName+"("+author.replace("作者：","")+")手机阅读,小说"+bookName+"TXT下载 - "+href; - param.put("content", content ); - param.put("styleid", "1"); - param.put("from", "share"); - param.put("appkey", "2351975812"); - param.put("refer", "http://www.zinglizingli.xyz/" + bookNum + "/"); - param.put("url_type", "0"); - param.put("visible", "0"); - //byte[] bytes = sendPostRequest(baseUrl, param); - - - MultiValueMap map = new LinkedMultiValueMap<>(); - map.setAll(param); - HttpHeaders headers = new HttpHeaders(); - headers.add("Accept", "*/*"); - headers.add("Accept-Encoding", "gzip, deflate"); - headers.add("Accept-Language", "zh-CN,zh;q=0.9"); - headers.add("Connection", "keep-alive"); - headers.add("Content-Length", "1146"); - headers.add("Content-Type", "application/x-www-form-urlencoded"); - - String[] cookieArr = cookieStr.split(";"); - List cookies = Arrays.asList(cookieArr); - headers.put(HttpHeaders.COOKIE, cookies); - - headers.add("Host", "service.weibo.com"); - headers.add("Origin", "http://service.weibo.com"); - headers.add("Referer", "http://service.weibo.com/share/share.php?appkey=2351975812&searchPic=true&title=%C2%A1%C2%BE%E4%BF%AE%E7%9C%9F%E8%81%8A%E5%A4%A9%E7%BE%A4%E6%9C%80%E6%96%B0%E7%AB%A0%E8%8A%82%E5%88%97%E8%A1%A8_%E4%BF%AE%E7%9C%9F%E8%81%8A%E5%A4%A9%E7%BE%A4%E6%9C%80%E6%96%B0%E7%AB%A0%E8%8A%82%E7%9B%AE%E5%BD%95_%E9%85%B8%E5%91%B3%E4%B9%A6%E5%B1%8B%C2%A1%C2%BF%E4%BF%AE%E7%9C%9F%E8%81%8A%E5%A4%A9%E7%BE%A4%E6%9C%80%E6%96%B0%E7%AB%A0%E8%8A%82%E7%94%B1%E7%BD%91%E5%8F%8B%E6%8F%90%E4%BE%9B%EF%BC%8C%E3%80%8A%E4%BF%AE%E7%9C%9F%E8%81%8A%E5%A4%A9%E7%BE%A4%E3%80%8B%E6%83%85%E8%8A%82%E8%B7%8C%E5%AE%95%E8%B5%B7%E4%BC%8F%E3%80%81%E6%89%A3%E4%BA%BA%E5%BF%83%E5%BC%A6%EF%BC%8C%E6%98%AF%E4%B8%80%E6%9C%AC%E6%83%85%E8%8A%82%E4%B8%8E%E6%96%87%E7%AC%94%E4%BF%B1%E4%BD%B3%E7%9A%84%E9%83%BD%E5%B8%82%E5%B0%8F%E8%AF%B4%E5%B0%8F%E8%AF%B4%EF%BC%8C%E9%85%B8%E5%91%B3%E4%B9%A6%E5%B1%8B%E5%85%8D%E8%B4%B9%E6%8F%90%E4%BE%9B%E5%94%90%E7%A0%96%E6%9C%80%E6%96%B0%E6%B8%85%E7%88%BD%E5%B9%B2%E5%87%80%E7%9A%84%E6%96%87%E5%AD%97%E7%AB%A0%E8%8A%82%E5%9C%A8%E7%BA%BF%E9%98%85%E8%AF%BB.&url=http%3A//www.zinglizingli.xyz/" + bookNum + "/"); - headers.add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE"); - headers.add("X-Requested-With", "XMLHttpRequest"); - headers.add("Accept-Encoding", "gzip, deflate"); - headers.add("Accept-Language", "zh-CN,zh;q=0.9"); - headers.add("Connection", "keep-alive"); - headers.add("Content-Length", "1146"); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - HttpEntity> request = new HttpEntity<>(map, headers); - - ResponseEntity stringResponseEntity = template.postForEntity(baseUrl, request, String.class, map); - - return stringResponseEntity.getBody(); - } - - -} diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/utils/Constants.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/Constants.java new file mode 100644 index 0000000..62e7474 --- /dev/null +++ b/novel-front/src/main/java/xyz/zinglizingli/common/utils/Constants.java @@ -0,0 +1,54 @@ +package xyz.zinglizingli.common.utils; + +/** + * @author XXY + */ +public class Constants { + + /** + * 服务异常路径 + * */ + public static final String SERVER_ERROR_PATH = "/mang.html"; + + /** + * 404跳转路径 + * */ + public static final String NOT_FOUND_PATH = "/"; + + /** + * 本地图片保存前缀 + * */ + public static final String LOCAL_PIC_PREFIX = "/localPic/"; + + /** + * 漫画内容访问前缀 + * */ + public static final String CARTOON_CONTENT_PREFIX = "/manhua/"; + + /** + * 漫画图片访问前缀 + * */ + public static final String CARTOON_PIC_PREFIX = "/manhua/images/"; + + /** + * 漫画图片访问前缀 + * */ + public static final String CARTOON_STATIC_PREFIX = "/manhua/statics/"; + + /** + * 爬取的动漫网站访问前缀 + * */ + public static final String CRAWL_CARTOON_URL_PREFIX = "https://www.dmzj.com/"; + + /** + * 爬取的动漫网站图片访问前缀 + * */ + public static final String CRAWL_CARTOON_PIC_URL_PREFIX = "https://images.dmzj.com/"; + + /** + * 爬取的动漫网站静态文件访问前缀 + * */ + public static final String CRAWL_CARTOON_STATIC_URL_PREFIX = "https://static.dmzj.com/"; + + +} diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/utils/ContentFactory.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/ContentFactory.java deleted file mode 100644 index 2288057..0000000 --- a/novel-front/src/main/java/xyz/zinglizingli/common/utils/ContentFactory.java +++ /dev/null @@ -1,65 +0,0 @@ -package xyz.zinglizingli.common.utils; - -import java.util.*; - -public class ContentFactory { - - - private static String[] hotWords = {"雪鹰领主是我吃西红柿写作的一本非常经典的玄幻小说,本站免费提供雪鹰领主最新最全的TXT全文小说阅读。", - "飞剑问道是我吃西红柿的经典仙侠小说作品，本站提供我吃西红柿TXT小说免费阅读。","" + - "终极美女保镖小说阅读，《终极美女保镖》情节跌宕起伏、扣人心弦，是一本情节与文笔俱佳的玄幻小说，本站免费提供终极美女保镖最新清爽干净的文字章节在线阅读。", - "一世倾城：冷宫弃妃,一世倾城：冷宫弃妃小说无弹窗阅读,一世倾城：冷宫弃妃小说无广告全文阅读,作者冷青衫", - "校花之贴身高手最新章节，VIP章节免费阅读，校花之贴身高手由寂无倾情打造。看精品小说，上看小说吧！", - "邪王追妻最新章节_邪王追妻免费全文阅读", - "《最强反派系统》免费阅读，最强反派系统小说最新章节免费阅读，VIP章节免费阅读。", - "大妖尊小说试阅读,大妖尊,大妖尊全文阅读,大妖尊最新章节", - "诡秘之主，诡秘之主小说阅读。玄幻小说诡秘之主由作家爱潜水的乌贼创作，看小说吧提供诡秘之主首发最新章节及章节列表，诡秘之主最新更新尽在看小说吧", - "全球高武，全球高武小说免费阅读。都市小说全球高武由作家老鹰吃小鸡创作，本站提供全球高武小说首发最新免费章节及章节列表，全球高武小说最新更新免费阅读","" + - "前任无双由作家跃千愁创作，前任无双小说免费阅读，无双首发最新章节及章节列表免费阅读，无弹窗、无广告小说免费月的", - "万古第一神小说是由作家风青阳所著的东方玄幻小说，本站提供万古第一神小说小说最新章节免费阅读", - "《剑来》小说是烽火戏诸侯在著作的武侠仙侠, 本站提供无广告、无弹窗的剑来小说最新章节全文免费阅读,请随时关注更新最快的小说阅读网看小说吧。", - "免费小说阅读,精彩免费小说最新章节尽在看小说吧,小说,小说网；网络小说；小说下载；小说txt,小说全文阅读，无弹窗广告尽在本站", - "看小说吧收集、整理和免费分享作者几人哀愁的最新免费小说：回到过去当女神最新目录列表，回到过去当女神小说是属于网友受欢迎的奇幻玄幻类型的小说。", - "大主宰无弹窗无广告最新免费章节由网友提供，《大主宰》小说情节跌宕起伏、扣人心弦：是一本情节与文笔俱佳的玄幻小说，看小说吧小说网免费大主宰最新免费的清爽干净的文字VIP章节在线阅读。", - "旋风少女小说最新免费章节由网友提供，《旋风少女》是一本受欢迎的情节和文笔俱佳的都市小说，旋风少女无弹窗无广告小说最新章节目录免费阅读，旋风少女VIP章节免费阅读！", - "斗破苍穹最新章节无弹窗是天蚕土豆倾力打造的一本非常耐读的小说，情节波澜起伏，由浅入深，层层推进，希望你能喜欢本书，支持天蚕土豆请收藏并推荐，斗破苍穹无弹窗最新免费章节目录免费提供" - ,"九星霸体诀是平凡魔术师写作的一本非常经典的玄幻小说,本站免费提供平凡魔术师最新最全的TXT全文小说阅读。", - "万古神帝是飞天鱼的经典仙侠小说作品，本站提供万古神帝TXT小说免费阅读。","" + - "无敌真寂寞小说阅读，《无敌真寂寞》情节跌宕起伏、扣人心弦，是一本情节与文笔俱佳的玄幻小说，本站免费提供无敌真寂寞最新清爽干净的文字章节在线阅读。", - "绝鼎丹尊,绝鼎丹尊小说无弹窗阅读,绝鼎丹尊小说无广告全文阅读,作者万古青莲", - "龙王传说最新章节，VIP章节免费阅读，龙王传说由唐家三少打造。看精品小说，上看小说吧！", - "人道至尊，人道至尊小说阅读。玄幻小说人道至尊由作家宅猪创作，看小说吧提供人道至尊首发最新章节及章节列表，人道至尊最新更新尽在看小说吧小说网", - "还是地球人狠，还是地球人狠免费阅读。都市小说全球高武由作家剑舞秀创作，本站提供还是地球人狠小说首发最新免费章节及章节列表，全球高武小说最新更新免费阅读","" + - "《求魔》小说是在耳根著作的武侠仙侠, 本站提供无广告、无弹窗的求魔小说最新章节全文免费阅读,请随时关注更新最快的小说阅读网看小说吧。", - "看小说吧收集、整理和免费分享作者唐家三少的最新免费小说：天火大道最新目录列表，天火大道小说是属于网友受欢迎的奇幻玄幻类型的小说。", - "饲养全人类无弹窗无广告最新免费章节由网友提供，《饲养全人类》小说情节跌宕起伏、扣人心弦：是一本情节与文笔俱佳的玄幻小说，看小说吧小说网免费饲养全人类最新免费的清爽干净的文字VIP章节在线阅读。", - "方外：消失的八门小说最新免费章节由网友提供，《方外：消失的八门》是一本受欢迎的情节和文笔俱佳的都市小说，方外：消失的八门无弹窗无广告小说最新章节目录免费阅读，方外：消失的八门VIP章节免费阅读！", - "武林赘婿最新章节无弹窗是左山左行倾力打造的一本非常耐读的小说，情节波澜起伏，由浅入深，层层推进，希望你能喜欢本书，支持武林赘婿请收藏并推荐，武林赘婿无弹窗最新免费章节目录免费提供"}; - - - - public static Map giveRandomContent(){ - Map contentMap = new HashMap<>(); - int size = 0; - - for(int i = 0 ; i < hotWords.length ; i++){ - if(size >= 18){ - break; - } - String value = hotWords[new Random().nextInt(hotWords.length)]; - if(contentMap.values().contains(value)){ - continue; - } - contentMap.put("hotWord_"+size,value); - size++; - } - if(size < 18){ - for(int i = 0 ; i< 18 - size ; i++){ - contentMap.put("hotWord_"+size,hotWords[new Random().nextInt(hotWords.length)]); - size++; - } - } - return contentMap; - - } -} diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/utils/RestTemplateUtil.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/RestTemplateUtil.java index 2dd180b..b956290 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/utils/RestTemplateUtil.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/utils/RestTemplateUtil.java @@ -6,23 +6,31 @@ import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.web.client.RestTemplate; import java.nio.charset.Charset; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class RestTemplateUtil { + private static Map restTemplateMap = new HashMap<>(); - public static RestTemplate getInstance(String charset) { - HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory(); - httpRequestFactory.setConnectionRequestTimeout(3000); - httpRequestFactory.setConnectTimeout(3000); - httpRequestFactory.setReadTimeout(10000); - RestTemplate restTemplate = new RestTemplate(httpRequestFactory); - List> list = restTemplate.getMessageConverters(); - for (HttpMessageConverter httpMessageConverter : list) { - if(httpMessageConverter instanceof StringHttpMessageConverter) { - ((StringHttpMessageConverter) httpMessageConverter).setDefaultCharset(Charset.forName(charset)); - break; + public static RestTemplate getInstance(Charset charset) { + RestTemplate restTemplate = restTemplateMap.get(charset.name()); + if(restTemplate == null) { + + HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory(); + httpRequestFactory.setConnectionRequestTimeout(3000); + httpRequestFactory.setConnectTimeout(3000); + httpRequestFactory.setReadTimeout(10000); + restTemplate = new RestTemplate(httpRequestFactory); + List> list = restTemplate.getMessageConverters(); + for (HttpMessageConverter httpMessageConverter : list) { + if (httpMessageConverter instanceof StringHttpMessageConverter) { + ((StringHttpMessageConverter) httpMessageConverter).setDefaultCharset(charset); + break; + } } + restTemplateMap.put(charset.name(),restTemplate); } return restTemplate; } diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/utils/SpringUtil.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/SpringUtil.java index 53d452f..10804a2 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/utils/SpringUtil.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/utils/SpringUtil.java @@ -8,23 +8,17 @@ import org.springframework.stereotype.Component; import java.io.UnsupportedEncodingException; +/** + * @author XXY + */ @Component public class SpringUtil implements ApplicationContextAware { - public static void main(String[] args) { - String a = "���"; - try { - String b = new String(a.getBytes("ISO-8859-1"),"gbk"); - System.out.println(b); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - private static ApplicationContext applicationContext; + @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { if (SpringUtil.applicationContext == null) { SpringUtil.applicationContext = applicationContext; diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/web/IndexController.java b/novel-front/src/main/java/xyz/zinglizingli/common/web/IndexController.java index bd5749f..473f554 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/web/IndexController.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/web/IndexController.java @@ -1,6 +1,7 @@ package xyz.zinglizingli.common.web; +import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; @@ -14,19 +15,21 @@ import xyz.zinglizingli.common.config.IndexRecBooksConfig; import java.util.List; import java.util.Map; +/** + * 首页controller + * @author XXY + */ @Controller @RequestMapping +@RequiredArgsConstructor public class IndexController { - @Autowired - private BookService bookService; + private final BookService bookService; - @Autowired - private CommonCacheUtil commonCacheUtil; + private final CommonCacheUtil commonCacheUtil; - @Autowired - private IndexRecBooksConfig indexRecBooksConfig; + private final IndexRecBooksConfig indexRecBooksConfig; @@ -49,6 +52,7 @@ public class IndexController { hotBooks = bookService.search(1, 9, null, null, null, null, null, null, null, "visit_count DESC,score ", "DESC"); commonCacheUtil.setObject(CacheKeyConstans.HOT_BOOK_LIST_KEY, hotBooks, 60 * 60 * 24); } + List newBooks = (List) commonCacheUtil.getObject(CacheKeyConstans.NEWST_BOOK_LIST_KEY); if (newBooks == null) { //查询最近更新数据