201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-06-24 12:46:38 +00:00
Code Issues Packages Releases Wiki Activity

refactor: 引入xxy-common相关依赖

Browse Source
This commit is contained in:
xiaoyang
2021-12-11 18:30:22 +08:00
parent 776083076c
commit 16e4c98a45
45 changed files with 252 additions and 899 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java
View File

@ -1,10 +1,11 @@
package com.java2nb.novel.controller;
import com.java2nb.novel.core.bean.PageBean;
import com.java2nb.novel.core.bean.ResultBean;
import io.github.xxyopen.model.page.PageBean;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.service.CrawlService;
import io.github.xxyopen.model.resp.RestResult;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@ -23,10 +24,10 @@ public class CrawlController {
* 新增爬虫源
* */
@PostMapping("addCrawlSource")
public ResultBean<Void> addCrawlSource(CrawlSource source){
public RestResult<Void> addCrawlSource(CrawlSource source){
crawlService.addCrawlSource(source);
return ResultBean.ok();
return RestResult.ok();
}
@ -34,30 +35,30 @@ public class CrawlController {
* 爬虫源分页列表查询
* */
@GetMapping("listCrawlByPage")
public ResultBean<PageBean<CrawlSource>> listCrawlByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
public RestResult<PageBean<CrawlSource>> listCrawlByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
return ResultBean.ok(crawlService.listCrawlByPage(page,pageSize));
return RestResult.ok(crawlService.listCrawlByPage(page,pageSize));
}
/**
* 开启或停止爬虫
* */
@PostMapping("openOrCloseCrawl")
public ResultBean<Void> openOrCloseCrawl(Integer sourceId,Byte sourceStatus){
public RestResult<Void> openOrCloseCrawl(Integer sourceId,Byte sourceStatus){
crawlService.openOrCloseCrawl(sourceId,sourceStatus);
return ResultBean.ok();
return RestResult.ok();
}
/**
* 新增单本采集任务
* */
@PostMapping("addCrawlSingleTask")
public ResultBean<Void> addCrawlSingleTask(CrawlSingleTask singleTask){
public RestResult<Void> addCrawlSingleTask(CrawlSingleTask singleTask){
crawlService.addCrawlSingleTask(singleTask);
return ResultBean.ok();
return RestResult.ok();
}
@ -65,20 +66,20 @@ public class CrawlController {
* 单本采集任务分页列表查询
* */
@GetMapping("listCrawlSingleTaskByPage")
public ResultBean<PageBean<CrawlSingleTask>> listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
public RestResult<PageBean<CrawlSingleTask>> listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
return ResultBean.ok(crawlService.listCrawlSingleTaskByPage(page,pageSize));
return RestResult.ok(crawlService.listCrawlSingleTaskByPage(page,pageSize));
}
/**
* 删除采集任务
* */
@DeleteMapping("delCrawlSingleTask/{id}")
public ResultBean<Void> delCrawlSingleTask(@PathVariable("id") Long id){
public RestResult<Void> delCrawlSingleTask(@PathVariable("id") Long id){
crawlService.delCrawlSingleTask(id);
return ResultBean.ok();
return RestResult.ok();
}

3
novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java
View File

@ -5,6 +5,7 @@ import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.utils.Constants;
import io.github.xxyopen.util.IdWorker;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -26,7 +27,7 @@ import static java.util.regex.Pattern.compile;
@Slf4j
public class CrawlParser {
private static final IdWorker idWorker = new IdWorker();
private static final IdWorker idWorker = IdWorker.INSTANCE;
private static final RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");

2
novel-crawl/src/main/java/com/java2nb/novel/core/schedule/CrawlThreadMonitor.java
View File

@ -3,9 +3,9 @@ package com.java2nb.novel.core.schedule;
import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.utils.ThreadUtil;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.service.CrawlService;
import io.github.xxyopen.util.ThreadUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;

2
novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java
View File

@ -1,6 +1,6 @@
package com.java2nb.novel.service;
import com.java2nb.novel.core.bean.PageBean;
import io.github.xxyopen.model.page.PageBean;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;

19
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -2,17 +2,17 @@ package com.java2nb.novel.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.pagehelper.PageHelper;
import com.java2nb.novel.core.bean.PageBean;
import io.github.xxyopen.model.page.PageBean;
import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.crawl.CrawlParser;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.core.enums.ResponseStatus;
import com.java2nb.novel.core.exception.BusinessException;
import com.java2nb.novel.core.utils.BeanUtil;
import com.java2nb.novel.core.utils.IdWorker;
import com.java2nb.novel.core.utils.SpringUtil;
import com.java2nb.novel.core.utils.ThreadUtil;
import io.github.xxyopen.model.page.builder.pagehelper.PageBuilder;
import io.github.xxyopen.util.IdWorker;
import io.github.xxyopen.util.ThreadUtil;
import io.github.xxyopen.web.exception.BusinessException;
import io.github.xxyopen.web.util.BeanUtil;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
@ -24,6 +24,7 @@ import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.vo.CrawlSingleTaskVO;
import com.java2nb.novel.vo.CrawlSourceVO;
import io.github.xxyopen.web.util.SpringUtil;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -79,7 +80,7 @@ public class CrawlServiceImpl implements CrawlService {
.build()
.render(RenderingStrategies.MYBATIS3);
List<CrawlSource> crawlSources = crawlSourceMapper.selectMany(render);
PageBean<CrawlSource> pageBean = new PageBean<>(crawlSources);
PageBean<CrawlSource> pageBean = PageBuilder.build(crawlSources);
pageBean.setList(BeanUtil.copyList(crawlSources, CrawlSourceVO.class));
return pageBean;
}
@ -168,7 +169,7 @@ public class CrawlServiceImpl implements CrawlService {
.build()
.render(RenderingStrategies.MYBATIS3);
List<CrawlSingleTask> crawlSingleTasks = crawlSingleTaskMapper.selectMany(render);
PageBean<CrawlSingleTask> pageBean = new PageBean<>(crawlSingleTasks);
PageBean<CrawlSingleTask> pageBean = PageBuilder.build(crawlSingleTasks);
pageBean.setList(BeanUtil.copyList(crawlSingleTasks, CrawlSingleTaskVO.class));
return pageBean;
}
@ -299,7 +300,7 @@ public class CrawlServiceImpl implements CrawlService {
book.setCrawlBookId(bookId);
book.setCrawlSourceId(sourceId);
book.setCrawlLastTime(new Date());
book.setId(new IdWorker().nextId());
book.setId(IdWorker.INSTANCE.nextId());
//解析章节目录
CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0), chapter -> {
bookService.saveBookAndIndexAndContent(book, chapter.getBookIndexList(), chapter.getBookContentList());