201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-06-24 12:46:38 +00:00
Code Issues Packages Releases Wiki Activity

部分代码重构

Browse Source
This commit is contained in:
xiaoyang
2021-08-16 15:42:56 +08:00
parent 4939bcf418
commit 3520200a87
28 changed files with 234 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java
View File

@ -1,13 +1,10 @@
package com.java2nb.novel.controller;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.core.bean.PageBean;
import com.java2nb.novel.core.bean.ResultBean;
import com.java2nb.novel.core.utils.BeanUtil;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.vo.CrawlSingleTaskVO;
import com.java2nb.novel.vo.CrawlSourceVO;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@ -26,7 +23,7 @@ public class CrawlController {
* 新增爬虫源
* */
@PostMapping("addCrawlSource")
public ResultBean addCrawlSource(CrawlSource source){
public ResultBean<Void> addCrawlSource(CrawlSource source){
crawlService.addCrawlSource(source);
return ResultBean.ok();
@ -37,7 +34,7 @@ public class CrawlController {
* 爬虫源分页列表查询
* */
@GetMapping("listCrawlByPage")
public ResultBean listCrawlByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
public ResultBean<PageBean<CrawlSource>> listCrawlByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
return ResultBean.ok(crawlService.listCrawlByPage(page,pageSize));
}
@ -46,7 +43,7 @@ public class CrawlController {
* 开启或停止爬虫
* */
@PostMapping("openOrCloseCrawl")
public ResultBean openOrCloseCrawl(Integer sourceId,Byte sourceStatus){
public ResultBean<Void> openOrCloseCrawl(Integer sourceId,Byte sourceStatus){
crawlService.openOrCloseCrawl(sourceId,sourceStatus);
@ -57,7 +54,7 @@ public class CrawlController {
* 新增单本采集任务
* */
@PostMapping("addCrawlSingleTask")
public ResultBean addCrawlSingleTask(CrawlSingleTask singleTask){
public ResultBean<Void> addCrawlSingleTask(CrawlSingleTask singleTask){
crawlService.addCrawlSingleTask(singleTask);
return ResultBean.ok();
@ -68,7 +65,7 @@ public class CrawlController {
* 单本采集任务分页列表查询
* */
@GetMapping("listCrawlSingleTaskByPage")
public ResultBean listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
public ResultBean<PageBean<CrawlSingleTask>> listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
return ResultBean.ok(crawlService.listCrawlSingleTaskByPage(page,pageSize));
}
@ -77,7 +74,7 @@ public class CrawlController {
* 删除采集任务
* */
@DeleteMapping("delCrawlSingleTask/{id}")
public ResultBean delCrawlSingleTask(@PathVariable("id") Long id){
public ResultBean<Void> delCrawlSingleTask(@PathVariable("id") Long id){
crawlService.delCrawlSingleTask(id);

7
novel-crawl/src/main/java/com/java2nb/novel/controller/PageController.java
View File

@ -1,18 +1,11 @@
package com.java2nb.novel.controller;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.entity.News;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import java.util.List;
/**
* @author 11797
*/

2
novel-crawl/src/main/java/com/java2nb/novel/core/crawl/RuleBean.java
View File

@ -35,7 +35,7 @@ public class RuleBean {
private String statusPatten;
private String scorePatten;
private String visitCountPatten;
private String descStart;;
private String descStart;
private String descEnd;
private String updateTimePatten;
private String updateTimeFormatPatten;

2
novel-crawl/src/main/java/com/java2nb/novel/core/schedule/CrawlThreadMonitor.java
View File

@ -8,12 +8,10 @@ import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.service.CrawlService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**

1
novel-crawl/src/main/java/com/java2nb/novel/service/BookService.java
View File

@ -72,6 +72,7 @@ public interface BookService {
* 更新一下最后一次的抓取时间
* @param bookId 小说ID
* */
@Deprecated
void updateCrawlLastTime(Long bookId);
/**

4
novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java
View File

@ -1,11 +1,9 @@
package com.java2nb.novel.service;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.core.bean.PageBean;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.vo.CrawlSourceVO;
import java.util.List;
@ -64,7 +62,7 @@ public interface CrawlService {
* 根据分类ID和规则解析分类列表
* @param catId 分类ID
* @param ruleBean 规则对象
* @param sourceId
* @param sourceId 爬虫源ID
*/
void parseBookList(int catId, RuleBean ruleBean, Integer sourceId);

15
novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -176,19 +176,4 @@ public class BookServiceImpl implements BookService {
}
/**
* 查询最后的章节
* */
private BookIndex queryLastIndex(Long bookId) {
return bookIndexMapper.queryLastIndex(bookId);
}
/**
* 查询小说总字数
* */
private Integer queryTotalWordCount(Long bookId) {
return bookMapper.queryTotalWordCount(bookId);
}
}

22
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -1,9 +1,7 @@
package com.java2nb.novel.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.pagehelper.Page;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.core.bean.PageBean;
import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
@ -16,9 +14,13 @@ import com.java2nb.novel.core.utils.BeanUtil;
import com.java2nb.novel.core.utils.IdWorker;
import com.java2nb.novel.core.utils.SpringUtil;
import com.java2nb.novel.core.utils.ThreadUtil;
import com.java2nb.novel.entity.*;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.mapper.*;
import com.java2nb.novel.mapper.CrawlSingleTaskDynamicSqlSupport;
import com.java2nb.novel.mapper.CrawlSingleTaskMapper;
import com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport;
import com.java2nb.novel.mapper.CrawlSourceMapper;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.vo.CrawlSingleTaskVO;
@ -30,18 +32,14 @@ import org.apache.commons.lang3.StringUtils;
import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClient;
import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClientWithChrome;
import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlBookId;
import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlSourceId;
import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*;
import static org.mybatis.dynamic.sql.SqlBuilder.*;
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.select.SelectDSL.select;
/**
@ -122,11 +120,7 @@ public class CrawlServiceImpl implements CrawlService {
//按分类开始爬虫解析任务
for (int i = 1; i < 8; i++) {
final int catId = i;
Thread thread = new Thread(() -> {
parseBookList(catId, ruleBean, sourceId);
});
Thread thread = new Thread(() -> CrawlServiceImpl.this.parseBookList(catId, ruleBean, sourceId));
thread.start();
//thread加入到监控缓存中
threadIds.add(thread.getId());

3
novel-crawl/src/main/java/com/java2nb/novel/vo/CrawlSingleTaskVO.java
View File

@ -2,14 +2,15 @@ package com.java2nb.novel.vo;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.Date;
/**
* @author Administrator
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class CrawlSingleTaskVO extends CrawlSingleTask {

3
novel-crawl/src/main/java/com/java2nb/novel/vo/CrawlSourceVO.java
View File

@ -3,13 +3,14 @@ package com.java2nb.novel.vo;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.java2nb.novel.entity.CrawlSource;
import lombok.Data;
import lombok.EqualsAndHashCode;
import javax.annotation.Generated;
import java.util.Date;
/**
* @author Administrator
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class CrawlSourceVO extends CrawlSource{