201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-06-24 04:46:37 +00:00
Code Issues Packages Releases Wiki Activity

增加单本采集任务

Browse Source
This commit is contained in:
xiongxiaoyang
2020-06-15 15:08:15 +08:00
parent 3cbb6bf3fb
commit c9c714e71e
17 changed files with 738 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java
View File

@ -3,11 +3,12 @@ package com.java2nb.novel.controller;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.core.bean.ResultBean;
import com.java2nb.novel.core.utils.BeanUtil;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.vo.CrawlSingleTaskVO;
import com.java2nb.novel.vo.CrawlSourceVO;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@ -56,6 +57,38 @@ public class CrawlController {
return ResultBean.ok();
}
/**
* 新增单本采集任务
* */
@PostMapping("addCrawlSingleTask")
public ResultBean addCrawlSingleTask(CrawlSingleTask singleTask){
crawlService.addCrawlSingleTask(singleTask);
return ResultBean.ok();
}
/**
* 单本采集任务分页列表查询
* */
@PostMapping("listCrawlSingleTaskByPage")
public ResultBean listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
return ResultBean.ok(new PageInfo<>(BeanUtil.copyList(crawlService.listCrawlSingleTaskByPage(page,pageSize), CrawlSingleTaskVO.class)
));
}
/**
* 删除采集任务
* */
@PostMapping("delCrawlSingleTask")
public ResultBean delCrawlSingleTask(Long id){
crawlService.delCrawlSingleTask(id);
return ResultBean.ok();
}

49
novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java
View File

@ -3,10 +3,7 @@ package com.java2nb.novel.core.listener;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.java2nb.novel.core.crawl.CrawlParser;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.entity.*;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.utils.Constants;
@ -40,15 +37,15 @@ public class StarterListener implements ServletContextListener {
@Override
public void contextInitialized(ServletContextEvent sce) {
log.info("程序启动,开始执行自动更新线程。。。");
for(int i = 0 ; i<updateThreadCount; i++) {
for (int i = 0; i < updateThreadCount; i++) {
new Thread(() -> {
log.info("程序启动,开始执行自动更新线程。。。");
while (true) {
try {
//1.查询最新目录更新时间在一个月之内的前100条需要更新的数据
Date currentDate = new Date();
Date startDate = DateUtils.addDays(currentDate, -30);
List<Book> bookList ;
List<Book> bookList;
synchronized (this) {
bookList = bookService.queryNeedUpdateBook(startDate, 100);
}
@ -61,7 +58,7 @@ public class StarterListener implements ServletContextListener {
Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
//这里只做老书更新
book.setId(needUpdateBook.getId());
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
if (needUpdateBook.getPicUrl() != null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
//本地图片则不更新
book.setPicUrl(null);
}
@ -83,6 +80,42 @@ public class StarterListener implements ServletContextListener {
}
}).start();
}
new Thread(() -> {
log.info("程序启动,开始执行单本采集任务线程。。。");
while (true) {
CrawlSingleTask task = null;
byte crawlStatus = 0;
try {
//获取采集任务
task = crawlService.getCrawlSingleTask();
if (task != null) {
//查询爬虫规则
CrawlSource source = crawlService.queryCrawlSource(task.getSourceId());
RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
if (crawlService.parseBookAndSave(task.getCatId(), ruleBean, task.getSourceId(), task.getSourceBookId())) {
//采集成功
crawlStatus = 1;
}
}
Thread.sleep(1000 * 60);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
if (task != null) {
crawlService.updateCrawlSingleTask(task, crawlStatus);
}
}
}).start();
}
}

44
novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java
View File

@ -1,6 +1,7 @@
package com.java2nb.novel.service;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import java.util.List;
@ -39,6 +40,16 @@ public interface CrawlService {
* */
void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus);
/**
* 采集并保存小说
* @param catId 分类ID
* @param bookId 小说ID
* @param sourceId 源ID
* @param ruleBean 采集规则\
* @return true:成功，false:失败
* */
boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId);
/**
* 根据爬虫状态查询爬虫源集合
* @param sourceStatus 状态，0关闭，1开启
@ -61,4 +72,37 @@ public interface CrawlService {
* @return 源信息
* */
CrawlSource queryCrawlSource(Integer sourceId);
/**
* 新增单本采集任务
* @param singleTask 任务信息对象
* */
void addCrawlSingleTask(CrawlSingleTask singleTask);
/**
* 单本采集任务分页列表查询
* @param page 当前页码
* @param pageSize 分页大小
* @return 单本采集任务集合
* */
List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize);
/**
* 删除采集任务
* @param id 任务ID
* */
void delCrawlSingleTask(Long id);
/**
* 获取采集任务
* @return 采集任务
* */
CrawlSingleTask getCrawlSingleTask();
/**
* 更新单本采集任务
* @param task 采集任务
* @param status 采集状态
* */
void updateCrawlSingleTask(CrawlSingleTask task, Byte status);
}

134
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -6,12 +6,12 @@ import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.crawl.CrawlParser;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.core.enums.ResponseStatus;
import com.java2nb.novel.core.exception.BusinessException;
import com.java2nb.novel.core.utils.IdWorker;
import com.java2nb.novel.core.utils.SpringUtil;
import com.java2nb.novel.core.utils.ThreadUtil;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.entity.*;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.mapper.*;
import com.java2nb.novel.service.BookService;
@ -33,8 +33,7 @@ import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClient;
import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlBookId;
import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlSourceId;
import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*;
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.update;
import static org.mybatis.dynamic.sql.SqlBuilder.*;
import static org.mybatis.dynamic.sql.select.SelectDSL.select;
/**
@ -48,6 +47,8 @@ public class CrawlServiceImpl implements CrawlService {
private final CrawlSourceMapper crawlSourceMapper;
private final CrawlSingleTaskMapper crawlSingleTaskMapper;
private final BookService bookService;
@ -140,6 +141,62 @@ public class CrawlServiceImpl implements CrawlService {
return crawlSourceMapper.selectMany(render).get(0);
}
@Override
public void addCrawlSingleTask(CrawlSingleTask singleTask) {
if(bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(),singleTask.getAuthorName())){
throw new BusinessException(ResponseStatus.BOOK_EXISTS);
}
singleTask.setCreateTime(new Date());
crawlSingleTaskMapper.insertSelective(singleTask);
}
@Override
public List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize) {
PageHelper.startPage(page, pageSize);
SelectStatementProvider render = select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
.from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime.descending())
.build()
.render(RenderingStrategies.MYBATIS3);
return crawlSingleTaskMapper.selectMany(render);
}
@Override
public void delCrawlSingleTask(Long id) {
crawlSingleTaskMapper.deleteByPrimaryKey(id);
}
@Override
public CrawlSingleTask getCrawlSingleTask() {
List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
.from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
.where(CrawlSingleTaskDynamicSqlSupport.taskStatus,isEqualTo((byte)2))
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime)
.limit(1)
.build()
.render(RenderingStrategies.MYBATIS3));
return list.size() > 0 ? list.get(0) : null;
}
@Override
public void updateCrawlSingleTask(CrawlSingleTask task, Byte status) {
byte excCount = task.getExcCount();
excCount+=1;
task.setExcCount(excCount);
if(status == 1 || excCount == 5){
//当采集成功或者采集次数等于5，则更新采集最终状态，并停止采集
task.setTaskStatus(status);
}
crawlSingleTaskMapper.updateByPrimaryKeySelective(task);
}
/**
* 解析分类列表
*/
@ -173,35 +230,7 @@ public class CrawlServiceImpl implements CrawlService {
String bookId = bookIdMatcher.group(1);
Book book = CrawlParser.parseBook(ruleBean, bookId);
//这里只做新书入库，查询是否存在这本书
Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
//如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库
if (existBook == null) {
//没有该书，可以入库
book.setCatId(catId);
//根据分类ID查询分类
book.setCatName(bookService.queryCatNameByCatId(catId));
if (catId == 7) {
//女频
book.setWorkDirection((byte) 1);
} else {
//男频
book.setWorkDirection((byte) 0);
}
book.setCrawlBookId(bookId);
book.setCrawlSourceId(sourceId);
book.setCrawlLastTime(new Date());
book.setId(new IdWorker().nextId());
//解析章节目录
Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
} else {
//只更新书籍的爬虫相关字段
bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
}
parseBookAndSave(catId, ruleBean, sourceId, bookId);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
@ -232,6 +261,43 @@ public class CrawlServiceImpl implements CrawlService {
}
@Override
public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) {
Book book = CrawlParser.parseBook(ruleBean, bookId);
if(book.getBookName() == null || book.getAuthorName() == null){
return false;
}
//这里只做新书入库，查询是否存在这本书
Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
//如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库
if (existBook == null) {
//没有该书，可以入库
book.setCatId(catId);
//根据分类ID查询分类
book.setCatName(bookService.queryCatNameByCatId(catId));
if (catId == 7) {
//女频
book.setWorkDirection((byte) 1);
} else {
//男频
book.setWorkDirection((byte) 0);
}
book.setCrawlBookId(bookId);
book.setCrawlSourceId(sourceId);
book.setCrawlLastTime(new Date());
book.setId(new IdWorker().nextId());
//解析章节目录
Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
} else {
//只更新书籍的爬虫相关字段
bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
}
return true;
}
@Override
public void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus) {
CrawlSource source = new CrawlSource();

26
novel-crawl/src/main/java/com/java2nb/novel/vo/CrawlSingleTaskVO.java Normal file
View File

@ -0,0 +1,26 @@
package com.java2nb.novel.vo;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import lombok.Data;
import java.util.Date;
/**
* @author Administrator
*/
@Data
public class CrawlSingleTaskVO extends CrawlSingleTask {
@JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm")
private Date createTime;
@Override
public String toString() {
return super.toString();
}
}

4
novel-crawl/src/main/resources/static/css/user.css
View File

@ -10,7 +10,7 @@
.user_l .log_list { width:350px }
.user_l .s_input { margin-bottom:25px; font-size:14px }
.s_input { width:348px; height:30px; line-height:38px\9; vertical-align:middle; border:1px solid #ddd; border-radius:2px }
.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px; background:url(../images/icon_user.png) no-repeat 13px 13px }
.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px}
.icon_key { background-position: 13px -51px }
.icon_code { background-position: 13px -117px; width:200px; float:left }
.code_pic { height:38px; float:right }
@ -37,7 +37,7 @@
.fast_tit .title { background:#fff; font-size:16px; padding:3px 14px; position:relative; display:inline-block; z-index:999 }
/*userinfo*/
.my_l { width:198px; float:left; font-size: 13px; padding-top: 20px; }
.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff; background:url(../images/icon_user.png) no-repeat; margin-bottom:5px; color: #666 }
.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff; margin-bottom:5px; color: #666 }
.my_l li .on { background-color:#fafafa; border-left:2px solid #f80; color:#000; border-radius: 0 2px 2px 0 }
.my_l .link_1 { background-position:32px -188px }
.my_l .link_2 { background-position:32px -230px }

188
novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_add.html Normal file
View File

@ -0,0 +1,188 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
<title>爬虫管理系统-小说精品屋</title>
<link rel="stylesheet" href="/css/base.css?v=1"/>
<link rel="stylesheet" href="/css/user.css"/>
</head>
</head>
<body class="">
<div class="header">
<div class="mainNav" id="mainNav">
<div class="box_center cf"
style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
小说精品屋爬虫管理
</div>
</div>
</div>
<div class="main box_center cf">
<div class="userBox cf">
<div class="my_l">
<ul class="log_list">
<li><a class="link_1" href="/">爬虫源管理</a></li>
<li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>
</div>
<div class="my_r">
<div class="my_bookshelf">
<div class="userBox cf">
<form method="post" action="./register.html" id="form2">
<div class="user_l">
<div></div>
<h3>采集信息填写（示例均为笔趣阁：http://www.mcmssc.com）</h3>
<ul class="log_list">
<li><span id="LabErr"></span></li>
<input type="hidden" id="sourceName" class="s_input icon_name" >
<b>采集源:</b>
<li><select id="sourceId" class="s_input icon_key">
</select></li>
<b>采集分类:</b>
<li><select id="catId" class="s_input icon_key">
<option value="1">玄幻奇幻</option>
<option value="2">武侠仙侠</option>
<option value="3">都市言情</option>
<option value="4">历史军事</option>
<option value="5">科幻灵异</option>
<option value="6">网游竞技</option>
<option value="7">女生频道</option>
</select>
</li>
示例：<b>73_73911</b>
<li><input type="text" id="sourceBookId" class="s_input icon_key"
placeholder="采集的源站小说ID"/></li>
示例：<b>苏厨</b>
<li><input type="text" id="bookName" class="s_input icon_key"
placeholder="采集的小说名"/></li>
示例：<b>二子从周</b>
<li><input type="text" id="authorName" class="s_input icon_key"
placeholder="采集的小说作者名"/></li>
<!--示例：<b>https://m.xdingdiann.com/sort/0/1.html</b>
<li><input type="text" id="updateBookListUrl" class="s_input icon_key"
placeholder="小说更新列表url"></li>-->
<li><input type="button" onclick="addCrawlSingleTask()" name="btnRegister" value="提交"
id="btnRegister" class="btn_red"></li>
</ul>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
<script src="/layui/layui.all.js" type="text/javascript"></script>
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
$(function () {
$.ajax({
type: "POST",
url: "/crawl/listCrawlByPage",
data: {'curr':1,'limit':100},
dataType: "json",
success: function (data) {
if (data.code == 200) {
var crawlSourceList = data.data.list;
if (crawlSourceList.length > 0) {
var crawlSourceListHtml = "";
for(var i=0;i<crawlSourceList.length;i++){
var crawlSource = crawlSourceList[i];
crawlSourceListHtml+=("<option value=\""+crawlSource.id+"\">"+crawlSource.sourceName+"</option>");
}
$("#sourceId").html(crawlSourceListHtml);
}
} else if (data.code == 1001) {
//未登录
location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href);
}else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
});
function addCrawlSingleTask() {
var sourceId = $("#sourceId").find("option:selected").val();
var catId = $("#catId").find("option:selected").val();
var sourceName = $("#sourceId").find("option:selected").html();
var sourceBookId = $("#sourceBookId").val();
if (!sourceBookId){
layer.alert("采集的源站小说ID不能为空");
return;
}
var bookName = $("#bookName").val();
if (!bookName){
layer.alert("采集的小说名不能为空");
return;
}
var authorName = $("#authorName").val();
if (!authorName){
layer.alert("采集的小说作者名不能为空");
return;
}
$.ajax({
type: "POST",
url: "/crawl/addCrawlSingleTask",
data: {'sourceId': sourceId, 'sourceName': sourceName,'catId':catId,'sourceBookId' : sourceBookId,'bookName':bookName,'authorName':authorName},
dataType: "json",
success: function (data) {
if (data.code == 200) {
window.location.href = '/crawl/crawlSingleTask_list.html';
} else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
</script>
</html>

230
novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html Normal file
View File

@ -0,0 +1,230 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
<title>爬虫管理系统-小说精品屋</title>
<link rel="stylesheet" href="/css/base.css?v=1"/>
<link rel="stylesheet" href="/css/user.css"/>
</head>
</head>
<body class="">
<div class="header">
<div class="mainNav" id="mainNav">
<div class="box_center cf"
style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
小说精品屋爬虫管理
</div>
</div>
</div>
<div class="main box_center cf">
<div class="userBox cf">
<div class="my_l">
<ul class="log_list">
<li><a class="link_1" href="/">爬虫源管理</a></li>
<li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>
</div>
<div class="my_r">
<div class="my_bookshelf">
<div class="title cf">
<h2 class="fl">单本采集任务列表</h2>
<div class="fr"><a href="/crawl/crawlSingleTask_add.html" class="btn_red">增加单本采集任务</a></div>
</div>
<div id="divData" class="updateTable">
<table cellpadding="0" cellspacing="0">
<thead>
<tr>
<th class="style">
序号
</th>
<th class="name">
采集小说名
</th>
<th class="name">
采集小说作者名
</th>
<th class="goread">
采集次数
</th>
<th class="goread">
状态
</th>
<th class="name">
创建时间
</th>
<th class="goread">
操作
</th>
</tr>
</thead>
<tbody id="crawlSourceList">
</tbody>
</table>
<div class="pageBox cf" id="shellPage">
</div>
</div>
<!--<div id="divData" class="updateTable">
<table cellpadding="0" cellspacing="0">
<thead>
<tr>
<th class="name">
爬虫源（已开启的爬虫源）
</th>
<th class="chapter">
成功爬取数量（websocket实现）
</th>
<th class="time">
目标爬取数量
</th>
<th class="goread">
状态（正在运行，已停止）（一次只能运行一个爬虫源）
</th>
<th class="goread">
操作（启动，停止）
</th>
</tr>
</thead>
<tbody id="bookShelfList">
</tbody>
</table>
<div class="pageBox cf" id="shellPage">
</div>
</div>-->
</div>
</div>
</div>
</div>
</body>
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
<script src="/layui/layui.all.js" type="text/javascript"></script>
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
search(1, 10);
function search(curr, limit) {
$.ajax({
type: "POST",
url: "/crawl/listCrawlSingleTaskByPage",
data: {'curr': curr, 'limit': limit},
dataType: "json",
success: function (data) {
if (data.code == 200) {
var crawlSourceList = data.data.list;
if (crawlSourceList.length > 0) {
var crawlSourceListHtml = "";
for (var i = 0; i < crawlSourceList.length; i++) {
var crawlSource = crawlSourceList[i];
crawlSourceListHtml += (" <tr class=\"book_list\" vals=\"291\">\n" +
" <td class=\"style bookclass\">\n" +
" [" + (i + 1) + "]\n" +
" </td>\n" +
" <td class=\"name\">\n" +
" " + crawlSource.bookName + "\n" +
" </td>\n" +
" <td class=\"name\">\n" +
" " + crawlSource.authorName + "\n" +
" </td>\n" +
" <td class=\"goread\">\n" +
" " + crawlSource.excCount + "\n" +
" </td>\n" +
" <td class=\"goread\">\n" +
" " + (crawlSource.taskStatus == 0 ? '采集失败' : (crawlSource.taskStatus == 1 ? '采集成功' : (crawlSource.excCount>0?'采集中':'排队中'))) + "\n" +
" </td>\n" +
" <td class=\"name\" valsc=\"291|2037554|1\">"
+ crawlSource.createTime + "</td>\n" +
" <td class=\"goread\" id='opt" + crawlSource.id + "'><a href='javascript:del(" + crawlSource.id + ")'>删除 </a></td> </tr>");
}
$("#crawlSourceList").html(crawlSourceListHtml);
layui.use('laypage', function () {
var laypage = layui.laypage;
//执行一个laypage实例
laypage.render({
elem: 'shellPage' //注意，这里的 test1 是 ID，不用加 # 号
, count: data.data.total //数据总数，从服务端得到,
, curr: data.data.pageNum
, limit: data.data.pageSize
, jump: function (obj, first) {
//obj包含了当前分页的所有参数，比如：
console.log(obj.curr); //得到当前页，以便向服务端请求对应页的数据。
console.log(obj.limit); //得到每页显示的条数
//首次不执行
if (!first) {
search(obj.curr, obj.limit);
} else {
}
}
});
});
}
} else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
function del(id) {
$.ajax({
type: "POST",
url: "/crawl/delCrawlSingleTask",
data: {'id': id},
dataType: "json",
success: function (data) {
if (data.code == 200) {
location.reload();
} else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
</script>
</html>

1
novel-crawl/src/main/resources/templates/crawl/crawlSource_add.html
View File

@ -29,6 +29,7 @@
<ul class="log_list">
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>

1
novel-crawl/src/main/resources/templates/crawl/crawlSource_list.html
View File

@ -28,6 +28,7 @@
<ul class="log_list">
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>