mirror of
https://github.com/201206030/novel-plus.git
synced 2025-04-27 01:30:51 +00:00
增加单本采集任务
This commit is contained in:
xiongxiaoyang
parent
3cbb6bf3fb
commit
c9c714e71e
@ -55,9 +55,14 @@ public enum ResponseStatus {
|
||||
* */
|
||||
INVITE_CODE_INVALID(4001, "邀请码无效！"),
|
||||
AUTHOR_STATUS_FORBIDDEN(4002, "作者状态异常，暂不能管理小说！")
|
||||
, BOOKNAME_EXISTS(4003,"已发布过同名小说！")
|
||||
, BOOKNAME_EXISTS(4003,"已发布过同名小说！"),
|
||||
|
||||
,
|
||||
/**
|
||||
* 小说相关错误
|
||||
*/
|
||||
BOOK_EXISTS(5001,"该小说已存在")
|
||||
|
||||
,
|
||||
/**
|
||||
* 搜索引擎相关错误
|
||||
* */
|
||||
@ -68,6 +73,7 @@ public enum ResponseStatus {
|
||||
* 其他通用错误
|
||||
* */
|
||||
PASSWORD_ERROR(88001,"密码错误！");
|
||||
|
||||
private int code;
|
||||
private String msg;
|
||||
|
||||
|
||||
@ -1,84 +0,0 @@
|
||||
package com.java2nb.novel.entity;
|
||||
|
||||
import java.util.Date;
|
||||
import javax.annotation.Generated;
|
||||
|
||||
public class CrawlSingleTask {
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private Long id;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private Integer sourceId;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private String sourceBookId;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private Byte taskStatus;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private Byte excCount;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private Date createTime;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public void setId(Long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public Integer getSourceId() {
|
||||
return sourceId;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public void setSourceId(Integer sourceId) {
|
||||
this.sourceId = sourceId;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public String getSourceBookId() {
|
||||
return sourceBookId;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public void setSourceBookId(String sourceBookId) {
|
||||
this.sourceBookId = sourceBookId == null ? null : sourceBookId.trim();
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public Byte getTaskStatus() {
|
||||
return taskStatus;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public void setTaskStatus(Byte taskStatus) {
|
||||
this.taskStatus = taskStatus;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public Byte getExcCount() {
|
||||
return excCount;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public void setExcCount(Byte excCount) {
|
||||
this.excCount = excCount;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public Date getCreateTime() {
|
||||
return createTime;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public void setCreateTime(Date createTime) {
|
||||
this.createTime = createTime;
|
||||
}
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
package com.java2nb.novel.mapper;
|
||||
|
||||
import java.sql.JDBCType;
|
||||
import java.util.Date;
|
||||
import javax.annotation.Generated;
|
||||
import org.mybatis.dynamic.sql.SqlColumn;
|
||||
import org.mybatis.dynamic.sql.SqlTable;
|
||||
|
||||
public final class CrawlSingleTaskDynamicSqlSupport {
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final CrawlSingleTask crawlSingleTask = new CrawlSingleTask();
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<Long> id = crawlSingleTask.id;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<Integer> sourceId = crawlSingleTask.sourceId;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<String> sourceBookId = crawlSingleTask.sourceBookId;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<Byte> taskStatus = crawlSingleTask.taskStatus;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<Byte> excCount = crawlSingleTask.excCount;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<Date> createTime = crawlSingleTask.createTime;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final class CrawlSingleTask extends SqlTable {
|
||||
public final SqlColumn<Long> id = column("id", JDBCType.BIGINT);
|
||||
|
||||
public final SqlColumn<Integer> sourceId = column("source_id", JDBCType.INTEGER);
|
||||
|
||||
public final SqlColumn<String> sourceBookId = column("source_book_id", JDBCType.VARCHAR);
|
||||
|
||||
public final SqlColumn<Byte> taskStatus = column("task_status", JDBCType.TINYINT);
|
||||
|
||||
public final SqlColumn<Byte> excCount = column("exc_count", JDBCType.TINYINT);
|
||||
|
||||
public final SqlColumn<Date> createTime = column("create_time", JDBCType.TIMESTAMP);
|
||||
|
||||
public CrawlSingleTask() {
|
||||
super("crawl_single_task");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,200 +0,0 @@
|
||||
package com.java2nb.novel.mapper;
|
||||
|
||||
import static com.java2nb.novel.mapper.CrawlSingleTaskDynamicSqlSupport.*;
|
||||
import static org.mybatis.dynamic.sql.SqlBuilder.*;
|
||||
|
||||
import com.java2nb.novel.entity.CrawlSingleTask;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import javax.annotation.Generated;
|
||||
import org.apache.ibatis.annotations.DeleteProvider;
|
||||
import org.apache.ibatis.annotations.InsertProvider;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Result;
|
||||
import org.apache.ibatis.annotations.ResultMap;
|
||||
import org.apache.ibatis.annotations.Results;
|
||||
import org.apache.ibatis.annotations.SelectProvider;
|
||||
import org.apache.ibatis.annotations.UpdateProvider;
|
||||
import org.apache.ibatis.type.JdbcType;
|
||||
import org.mybatis.dynamic.sql.BasicColumn;
|
||||
import org.mybatis.dynamic.sql.delete.DeleteDSLCompleter;
|
||||
import org.mybatis.dynamic.sql.delete.render.DeleteStatementProvider;
|
||||
import org.mybatis.dynamic.sql.insert.render.InsertStatementProvider;
|
||||
import org.mybatis.dynamic.sql.insert.render.MultiRowInsertStatementProvider;
|
||||
import org.mybatis.dynamic.sql.select.CountDSLCompleter;
|
||||
import org.mybatis.dynamic.sql.select.SelectDSLCompleter;
|
||||
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
|
||||
import org.mybatis.dynamic.sql.update.UpdateDSL;
|
||||
import org.mybatis.dynamic.sql.update.UpdateDSLCompleter;
|
||||
import org.mybatis.dynamic.sql.update.UpdateModel;
|
||||
import org.mybatis.dynamic.sql.update.render.UpdateStatementProvider;
|
||||
import org.mybatis.dynamic.sql.util.SqlProviderAdapter;
|
||||
import org.mybatis.dynamic.sql.util.mybatis3.MyBatis3Utils;
|
||||
|
||||
@Mapper
|
||||
public interface CrawlSingleTaskMapper {
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceBookId, taskStatus, excCount, createTime);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
@SelectProvider(type=SqlProviderAdapter.class, method="select")
|
||||
long count(SelectStatementProvider selectStatement);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
@DeleteProvider(type=SqlProviderAdapter.class, method="delete")
|
||||
int delete(DeleteStatementProvider deleteStatement);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
@InsertProvider(type=SqlProviderAdapter.class, method="insert")
|
||||
int insert(InsertStatementProvider<CrawlSingleTask> insertStatement);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
@InsertProvider(type=SqlProviderAdapter.class, method="insertMultiple")
|
||||
int insertMultiple(MultiRowInsertStatementProvider<CrawlSingleTask> multipleInsertStatement);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
@SelectProvider(type=SqlProviderAdapter.class, method="select")
|
||||
@ResultMap("CrawlSingleTaskResult")
|
||||
Optional<CrawlSingleTask> selectOne(SelectStatementProvider selectStatement);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
@SelectProvider(type=SqlProviderAdapter.class, method="select")
|
||||
@Results(id="CrawlSingleTaskResult", value = {
|
||||
@Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
|
||||
@Result(column="source_id", property="sourceId", jdbcType=JdbcType.INTEGER),
|
||||
@Result(column="source_book_id", property="sourceBookId", jdbcType=JdbcType.VARCHAR),
|
||||
@Result(column="task_status", property="taskStatus", jdbcType=JdbcType.TINYINT),
|
||||
@Result(column="exc_count", property="excCount", jdbcType=JdbcType.TINYINT),
|
||||
@Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP)
|
||||
})
|
||||
List<CrawlSingleTask> selectMany(SelectStatementProvider selectStatement);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
@UpdateProvider(type=SqlProviderAdapter.class, method="update")
|
||||
int update(UpdateStatementProvider updateStatement);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default long count(CountDSLCompleter completer) {
|
||||
return MyBatis3Utils.countFrom(this::count, crawlSingleTask, completer);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default int delete(DeleteDSLCompleter completer) {
|
||||
return MyBatis3Utils.deleteFrom(this::delete, crawlSingleTask, completer);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default int deleteByPrimaryKey(Long id_) {
|
||||
return delete(c ->
|
||||
c.where(id, isEqualTo(id_))
|
||||
);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default int insert(CrawlSingleTask record) {
|
||||
return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c ->
|
||||
c.map(id).toProperty("id")
|
||||
.map(sourceId).toProperty("sourceId")
|
||||
.map(sourceBookId).toProperty("sourceBookId")
|
||||
.map(taskStatus).toProperty("taskStatus")
|
||||
.map(excCount).toProperty("excCount")
|
||||
.map(createTime).toProperty("createTime")
|
||||
);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default int insertMultiple(Collection<CrawlSingleTask> records) {
|
||||
return MyBatis3Utils.insertMultiple(this::insertMultiple, records, crawlSingleTask, c ->
|
||||
c.map(id).toProperty("id")
|
||||
.map(sourceId).toProperty("sourceId")
|
||||
.map(sourceBookId).toProperty("sourceBookId")
|
||||
.map(taskStatus).toProperty("taskStatus")
|
||||
.map(excCount).toProperty("excCount")
|
||||
.map(createTime).toProperty("createTime")
|
||||
);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default int insertSelective(CrawlSingleTask record) {
|
||||
return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c ->
|
||||
c.map(id).toPropertyWhenPresent("id", record::getId)
|
||||
.map(sourceId).toPropertyWhenPresent("sourceId", record::getSourceId)
|
||||
.map(sourceBookId).toPropertyWhenPresent("sourceBookId", record::getSourceBookId)
|
||||
.map(taskStatus).toPropertyWhenPresent("taskStatus", record::getTaskStatus)
|
||||
.map(excCount).toPropertyWhenPresent("excCount", record::getExcCount)
|
||||
.map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime)
|
||||
);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default Optional<CrawlSingleTask> selectOne(SelectDSLCompleter completer) {
|
||||
return MyBatis3Utils.selectOne(this::selectOne, selectList, crawlSingleTask, completer);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default List<CrawlSingleTask> select(SelectDSLCompleter completer) {
|
||||
return MyBatis3Utils.selectList(this::selectMany, selectList, crawlSingleTask, completer);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default List<CrawlSingleTask> selectDistinct(SelectDSLCompleter completer) {
|
||||
return MyBatis3Utils.selectDistinct(this::selectMany, selectList, crawlSingleTask, completer);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default Optional<CrawlSingleTask> selectByPrimaryKey(Long id_) {
|
||||
return selectOne(c ->
|
||||
c.where(id, isEqualTo(id_))
|
||||
);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default int update(UpdateDSLCompleter completer) {
|
||||
return MyBatis3Utils.update(this::update, crawlSingleTask, completer);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
static UpdateDSL<UpdateModel> updateAllColumns(CrawlSingleTask record, UpdateDSL<UpdateModel> dsl) {
|
||||
return dsl.set(id).equalTo(record::getId)
|
||||
.set(sourceId).equalTo(record::getSourceId)
|
||||
.set(sourceBookId).equalTo(record::getSourceBookId)
|
||||
.set(taskStatus).equalTo(record::getTaskStatus)
|
||||
.set(excCount).equalTo(record::getExcCount)
|
||||
.set(createTime).equalTo(record::getCreateTime);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
static UpdateDSL<UpdateModel> updateSelectiveColumns(CrawlSingleTask record, UpdateDSL<UpdateModel> dsl) {
|
||||
return dsl.set(id).equalToWhenPresent(record::getId)
|
||||
.set(sourceId).equalToWhenPresent(record::getSourceId)
|
||||
.set(sourceBookId).equalToWhenPresent(record::getSourceBookId)
|
||||
.set(taskStatus).equalToWhenPresent(record::getTaskStatus)
|
||||
.set(excCount).equalToWhenPresent(record::getExcCount)
|
||||
.set(createTime).equalToWhenPresent(record::getCreateTime);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default int updateByPrimaryKey(CrawlSingleTask record) {
|
||||
return update(c ->
|
||||
c.set(sourceId).equalTo(record::getSourceId)
|
||||
.set(sourceBookId).equalTo(record::getSourceBookId)
|
||||
.set(taskStatus).equalTo(record::getTaskStatus)
|
||||
.set(excCount).equalTo(record::getExcCount)
|
||||
.set(createTime).equalTo(record::getCreateTime)
|
||||
.where(id, isEqualTo(record::getId))
|
||||
);
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
default int updateByPrimaryKeySelective(CrawlSingleTask record) {
|
||||
return update(c ->
|
||||
c.set(sourceId).equalToWhenPresent(record::getSourceId)
|
||||
.set(sourceBookId).equalToWhenPresent(record::getSourceBookId)
|
||||
.set(taskStatus).equalToWhenPresent(record::getTaskStatus)
|
||||
.set(excCount).equalToWhenPresent(record::getExcCount)
|
||||
.set(createTime).equalToWhenPresent(record::getCreateTime)
|
||||
.where(id, isEqualTo(record::getId))
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -44,7 +44,7 @@
|
||||
</javaClientGenerator>
|
||||
|
||||
<!--生成全部表tableName设为%-->
|
||||
<table tableName="user_buy_record" domainObjectName="UserBuyRecord"/>
|
||||
<table tableName="crawl_single_task"/>
|
||||
|
||||
<!-- 指定数据库表 -->
|
||||
<!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->
|
||||
|
||||
@ -3,11 +3,12 @@ package com.java2nb.novel.controller;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.java2nb.novel.core.bean.ResultBean;
|
||||
import com.java2nb.novel.core.utils.BeanUtil;
|
||||
import com.java2nb.novel.entity.CrawlSingleTask;
|
||||
import com.java2nb.novel.entity.CrawlSource;
|
||||
import com.java2nb.novel.service.CrawlService;
|
||||
import com.java2nb.novel.vo.CrawlSingleTaskVO;
|
||||
import com.java2nb.novel.vo.CrawlSourceVO;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
@ -56,6 +57,38 @@ public class CrawlController {
|
||||
return ResultBean.ok();
|
||||
}
|
||||
|
||||
/**
|
||||
* 新增单本采集任务
|
||||
* */
|
||||
@PostMapping("addCrawlSingleTask")
|
||||
public ResultBean addCrawlSingleTask(CrawlSingleTask singleTask){
|
||||
crawlService.addCrawlSingleTask(singleTask);
|
||||
|
||||
return ResultBean.ok();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 单本采集任务分页列表查询
|
||||
* */
|
||||
@PostMapping("listCrawlSingleTaskByPage")
|
||||
public ResultBean listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
|
||||
|
||||
return ResultBean.ok(new PageInfo<>(BeanUtil.copyList(crawlService.listCrawlSingleTaskByPage(page,pageSize), CrawlSingleTaskVO.class)
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除采集任务
|
||||
* */
|
||||
@PostMapping("delCrawlSingleTask")
|
||||
public ResultBean delCrawlSingleTask(Long id){
|
||||
|
||||
crawlService.delCrawlSingleTask(id);
|
||||
|
||||
return ResultBean.ok();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -3,10 +3,7 @@ package com.java2nb.novel.core.listener;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.java2nb.novel.core.crawl.CrawlParser;
|
||||
import com.java2nb.novel.core.crawl.RuleBean;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.entity.BookContent;
|
||||
import com.java2nb.novel.entity.BookIndex;
|
||||
import com.java2nb.novel.entity.CrawlSource;
|
||||
import com.java2nb.novel.entity.*;
|
||||
import com.java2nb.novel.service.BookService;
|
||||
import com.java2nb.novel.service.CrawlService;
|
||||
import com.java2nb.novel.utils.Constants;
|
||||
@ -40,15 +37,15 @@ public class StarterListener implements ServletContextListener {
|
||||
|
||||
@Override
|
||||
public void contextInitialized(ServletContextEvent sce) {
|
||||
log.info("程序启动,开始执行自动更新线程。。。");
|
||||
for(int i = 0 ; i<updateThreadCount; i++) {
|
||||
for (int i = 0; i < updateThreadCount; i++) {
|
||||
new Thread(() -> {
|
||||
log.info("程序启动,开始执行自动更新线程。。。");
|
||||
while (true) {
|
||||
try {
|
||||
//1.查询最新目录更新时间在一个月之内的前100条需要更新的数据
|
||||
Date currentDate = new Date();
|
||||
Date startDate = DateUtils.addDays(currentDate, -30);
|
||||
List<Book> bookList ;
|
||||
List<Book> bookList;
|
||||
synchronized (this) {
|
||||
bookList = bookService.queryNeedUpdateBook(startDate, 100);
|
||||
}
|
||||
@ -61,7 +58,7 @@ public class StarterListener implements ServletContextListener {
|
||||
Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
|
||||
//这里只做老书更新
|
||||
book.setId(needUpdateBook.getId());
|
||||
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
|
||||
if (needUpdateBook.getPicUrl() != null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
|
||||
//本地图片则不更新
|
||||
book.setPicUrl(null);
|
||||
}
|
||||
@ -83,6 +80,42 @@ public class StarterListener implements ServletContextListener {
|
||||
|
||||
}
|
||||
}).start();
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
new Thread(() -> {
|
||||
log.info("程序启动,开始执行单本采集任务线程。。。");
|
||||
while (true) {
|
||||
CrawlSingleTask task = null;
|
||||
byte crawlStatus = 0;
|
||||
try {
|
||||
//获取采集任务
|
||||
task = crawlService.getCrawlSingleTask();
|
||||
|
||||
if (task != null) {
|
||||
//查询爬虫规则
|
||||
CrawlSource source = crawlService.queryCrawlSource(task.getSourceId());
|
||||
RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
|
||||
|
||||
if (crawlService.parseBookAndSave(task.getCatId(), ruleBean, task.getSourceId(), task.getSourceBookId())) {
|
||||
//采集成功
|
||||
crawlStatus = 1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Thread.sleep(1000 * 60);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
if (task != null) {
|
||||
crawlService.updateCrawlSingleTask(task, crawlStatus);
|
||||
}
|
||||
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package com.java2nb.novel.service;
|
||||
|
||||
import com.java2nb.novel.core.crawl.RuleBean;
|
||||
import com.java2nb.novel.entity.CrawlSingleTask;
|
||||
import com.java2nb.novel.entity.CrawlSource;
|
||||
|
||||
import java.util.List;
|
||||
@ -39,6 +40,16 @@ public interface CrawlService {
|
||||
* */
|
||||
void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus);
|
||||
|
||||
/**
|
||||
* 采集并保存小说
|
||||
* @param catId 分类ID
|
||||
* @param bookId 小说ID
|
||||
* @param sourceId 源ID
|
||||
* @param ruleBean 采集规则\
|
||||
* @return true:成功，false:失败
|
||||
* */
|
||||
boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId);
|
||||
|
||||
/**
|
||||
* 根据爬虫状态查询爬虫源集合
|
||||
* @param sourceStatus 状态，0关闭，1开启
|
||||
@ -61,4 +72,37 @@ public interface CrawlService {
|
||||
* @return 源信息
|
||||
* */
|
||||
CrawlSource queryCrawlSource(Integer sourceId);
|
||||
|
||||
/**
|
||||
* 新增单本采集任务
|
||||
* @param singleTask 任务信息对象
|
||||
* */
|
||||
void addCrawlSingleTask(CrawlSingleTask singleTask);
|
||||
|
||||
/**
|
||||
* 单本采集任务分页列表查询
|
||||
* @param page 当前页码
|
||||
* @param pageSize 分页大小
|
||||
* @return 单本采集任务集合
|
||||
* */
|
||||
List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize);
|
||||
|
||||
/**
|
||||
* 删除采集任务
|
||||
* @param id 任务ID
|
||||
* */
|
||||
void delCrawlSingleTask(Long id);
|
||||
|
||||
/**
|
||||
* 获取采集任务
|
||||
* @return 采集任务
|
||||
* */
|
||||
CrawlSingleTask getCrawlSingleTask();
|
||||
|
||||
/**
|
||||
* 更新单本采集任务
|
||||
* @param task 采集任务
|
||||
* @param status 采集状态
|
||||
* */
|
||||
void updateCrawlSingleTask(CrawlSingleTask task, Byte status);
|
||||
}
|
||||
|
||||
@ -6,12 +6,12 @@ import com.java2nb.novel.core.cache.CacheKey;
|
||||
import com.java2nb.novel.core.cache.CacheService;
|
||||
import com.java2nb.novel.core.crawl.CrawlParser;
|
||||
import com.java2nb.novel.core.crawl.RuleBean;
|
||||
import com.java2nb.novel.core.enums.ResponseStatus;
|
||||
import com.java2nb.novel.core.exception.BusinessException;
|
||||
import com.java2nb.novel.core.utils.IdWorker;
|
||||
import com.java2nb.novel.core.utils.SpringUtil;
|
||||
import com.java2nb.novel.core.utils.ThreadUtil;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.entity.BookContent;
|
||||
import com.java2nb.novel.entity.BookIndex;
|
||||
import com.java2nb.novel.entity.*;
|
||||
import com.java2nb.novel.entity.CrawlSource;
|
||||
import com.java2nb.novel.mapper.*;
|
||||
import com.java2nb.novel.service.BookService;
|
||||
@ -33,8 +33,7 @@ import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClient;
|
||||
import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlBookId;
|
||||
import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlSourceId;
|
||||
import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*;
|
||||
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
|
||||
import static org.mybatis.dynamic.sql.SqlBuilder.update;
|
||||
import static org.mybatis.dynamic.sql.SqlBuilder.*;
|
||||
import static org.mybatis.dynamic.sql.select.SelectDSL.select;
|
||||
|
||||
/**
|
||||
@ -48,6 +47,8 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
|
||||
private final CrawlSourceMapper crawlSourceMapper;
|
||||
|
||||
private final CrawlSingleTaskMapper crawlSingleTaskMapper;
|
||||
|
||||
private final BookService bookService;
|
||||
|
||||
|
||||
@ -140,6 +141,62 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
return crawlSourceMapper.selectMany(render).get(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addCrawlSingleTask(CrawlSingleTask singleTask) {
|
||||
|
||||
if(bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(),singleTask.getAuthorName())){
|
||||
throw new BusinessException(ResponseStatus.BOOK_EXISTS);
|
||||
|
||||
}
|
||||
singleTask.setCreateTime(new Date());
|
||||
crawlSingleTaskMapper.insertSelective(singleTask);
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize) {
|
||||
PageHelper.startPage(page, pageSize);
|
||||
SelectStatementProvider render = select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
|
||||
.from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
|
||||
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime.descending())
|
||||
.build()
|
||||
.render(RenderingStrategies.MYBATIS3);
|
||||
return crawlSingleTaskMapper.selectMany(render);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delCrawlSingleTask(Long id) {
|
||||
crawlSingleTaskMapper.deleteByPrimaryKey(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CrawlSingleTask getCrawlSingleTask() {
|
||||
|
||||
List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
|
||||
.from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
|
||||
.where(CrawlSingleTaskDynamicSqlSupport.taskStatus,isEqualTo((byte)2))
|
||||
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime)
|
||||
.limit(1)
|
||||
.build()
|
||||
.render(RenderingStrategies.MYBATIS3));
|
||||
|
||||
return list.size() > 0 ? list.get(0) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateCrawlSingleTask(CrawlSingleTask task, Byte status) {
|
||||
byte excCount = task.getExcCount();
|
||||
excCount+=1;
|
||||
task.setExcCount(excCount);
|
||||
if(status == 1 || excCount == 5){
|
||||
//当采集成功或者采集次数等于5，则更新采集最终状态，并停止采集
|
||||
task.setTaskStatus(status);
|
||||
}
|
||||
crawlSingleTaskMapper.updateByPrimaryKeySelective(task);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析分类列表
|
||||
*/
|
||||
@ -173,35 +230,7 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
|
||||
|
||||
String bookId = bookIdMatcher.group(1);
|
||||
Book book = CrawlParser.parseBook(ruleBean, bookId);
|
||||
//这里只做新书入库，查询是否存在这本书
|
||||
Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
|
||||
//如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库
|
||||
if (existBook == null) {
|
||||
//没有该书，可以入库
|
||||
book.setCatId(catId);
|
||||
//根据分类ID查询分类
|
||||
book.setCatName(bookService.queryCatNameByCatId(catId));
|
||||
if (catId == 7) {
|
||||
//女频
|
||||
book.setWorkDirection((byte) 1);
|
||||
} else {
|
||||
//男频
|
||||
book.setWorkDirection((byte) 0);
|
||||
}
|
||||
book.setCrawlBookId(bookId);
|
||||
book.setCrawlSourceId(sourceId);
|
||||
book.setCrawlLastTime(new Date());
|
||||
book.setId(new IdWorker().nextId());
|
||||
//解析章节目录
|
||||
Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
|
||||
|
||||
bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
|
||||
|
||||
} else {
|
||||
//只更新书籍的爬虫相关字段
|
||||
bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
|
||||
}
|
||||
parseBookAndSave(catId, ruleBean, sourceId, bookId);
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
@ -232,6 +261,43 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) {
|
||||
Book book = CrawlParser.parseBook(ruleBean, bookId);
|
||||
if(book.getBookName() == null || book.getAuthorName() == null){
|
||||
return false;
|
||||
}
|
||||
//这里只做新书入库，查询是否存在这本书
|
||||
Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
|
||||
//如果该小说不存在，则可以解析入库，但是标记该小说正在入库，30分钟之后才允许再次入库
|
||||
if (existBook == null) {
|
||||
//没有该书，可以入库
|
||||
book.setCatId(catId);
|
||||
//根据分类ID查询分类
|
||||
book.setCatName(bookService.queryCatNameByCatId(catId));
|
||||
if (catId == 7) {
|
||||
//女频
|
||||
book.setWorkDirection((byte) 1);
|
||||
} else {
|
||||
//男频
|
||||
book.setWorkDirection((byte) 0);
|
||||
}
|
||||
book.setCrawlBookId(bookId);
|
||||
book.setCrawlSourceId(sourceId);
|
||||
book.setCrawlLastTime(new Date());
|
||||
book.setId(new IdWorker().nextId());
|
||||
//解析章节目录
|
||||
Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
|
||||
|
||||
bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
|
||||
|
||||
} else {
|
||||
//只更新书籍的爬虫相关字段
|
||||
bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus) {
|
||||
CrawlSource source = new CrawlSource();
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
package com.java2nb.novel.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.java2nb.novel.entity.CrawlSingleTask;
|
||||
import com.java2nb.novel.entity.CrawlSource;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* @author Administrator
|
||||
*/
|
||||
@Data
|
||||
public class CrawlSingleTaskVO extends CrawlSingleTask {
|
||||
|
||||
@JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm")
|
||||
private Date createTime;
|
||||
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return super.toString();
|
||||
}
|
||||
}
|
||||
@ -10,7 +10,7 @@
|
||||
.user_l .log_list { width:350px }
|
||||
.user_l .s_input { margin-bottom:25px; font-size:14px }
|
||||
.s_input { width:348px; height:30px; line-height:38px\9; vertical-align:middle; border:1px solid #ddd; border-radius:2px }
|
||||
.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px; background:url(../images/icon_user.png) no-repeat 13px 13px }
|
||||
.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px}
|
||||
.icon_key { background-position: 13px -51px }
|
||||
.icon_code { background-position: 13px -117px; width:200px; float:left }
|
||||
.code_pic { height:38px; float:right }
|
||||
@ -37,7 +37,7 @@
|
||||
.fast_tit .title { background:#fff; font-size:16px; padding:3px 14px; position:relative; display:inline-block; z-index:999 }
|
||||
/*userinfo*/
|
||||
.my_l { width:198px; float:left; font-size: 13px; padding-top: 20px; }
|
||||
.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff; background:url(../images/icon_user.png) no-repeat; margin-bottom:5px; color: #666 }
|
||||
.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff; margin-bottom:5px; color: #666 }
|
||||
.my_l li .on { background-color:#fafafa; border-left:2px solid #f80; color:#000; border-radius: 0 2px 2px 0 }
|
||||
.my_l .link_1 { background-position:32px -188px }
|
||||
.my_l .link_2 { background-position:32px -230px }
|
||||
|
||||
@ -0,0 +1,188 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
||||
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
|
||||
<title>爬虫管理系统-小说精品屋</title>
|
||||
<link rel="stylesheet" href="/css/base.css?v=1"/>
|
||||
<link rel="stylesheet" href="/css/user.css"/>
|
||||
</head>
|
||||
</head>
|
||||
<body class="">
|
||||
|
||||
<div class="header">
|
||||
<div class="mainNav" id="mainNav">
|
||||
<div class="box_center cf"
|
||||
style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
|
||||
|
||||
小说精品屋爬虫管理
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="main box_center cf">
|
||||
<div class="userBox cf">
|
||||
<div class="my_l">
|
||||
|
||||
<ul class="log_list">
|
||||
<li><a class="link_1" href="/">爬虫源管理</a></li>
|
||||
<li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
|
||||
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
|
||||
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
<div class="my_r">
|
||||
<div class="my_bookshelf">
|
||||
|
||||
<div class="userBox cf">
|
||||
<form method="post" action="./register.html" id="form2">
|
||||
|
||||
<div class="user_l">
|
||||
<div></div>
|
||||
<h3>采集信息填写（示例均为笔趣阁：http://www.mcmssc.com）</h3>
|
||||
<ul class="log_list">
|
||||
<li><span id="LabErr"></span></li>
|
||||
<input type="hidden" id="sourceName" class="s_input icon_name" >
|
||||
<b>采集源:</b>
|
||||
<li><select id="sourceId" class="s_input icon_key">
|
||||
</select></li>
|
||||
<b>采集分类:</b>
|
||||
<li><select id="catId" class="s_input icon_key">
|
||||
<option value="1">玄幻奇幻</option>
|
||||
<option value="2">武侠仙侠</option>
|
||||
<option value="3">都市言情</option>
|
||||
<option value="4">历史军事</option>
|
||||
<option value="5">科幻灵异</option>
|
||||
<option value="6">网游竞技</option>
|
||||
<option value="7">女生频道</option>
|
||||
</select>
|
||||
</li>
|
||||
示例：<b>73_73911</b>
|
||||
<li><input type="text" id="sourceBookId" class="s_input icon_key"
|
||||
placeholder="采集的源站小说ID"/></li>
|
||||
示例：<b>苏厨</b>
|
||||
<li><input type="text" id="bookName" class="s_input icon_key"
|
||||
placeholder="采集的小说名"/></li>
|
||||
示例：<b>二子从周</b>
|
||||
<li><input type="text" id="authorName" class="s_input icon_key"
|
||||
placeholder="采集的小说作者名"/></li>
|
||||
|
||||
<!--示例：<b>https://m.xdingdiann.com/sort/0/1.html</b>
|
||||
<li><input type="text" id="updateBookListUrl" class="s_input icon_key"
|
||||
placeholder="小说更新列表url"></li>-->
|
||||
|
||||
|
||||
<li><input type="button" onclick="addCrawlSingleTask()" name="btnRegister" value="提交"
|
||||
id="btnRegister" class="btn_red"></li>
|
||||
</ul>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</body>
|
||||
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
|
||||
<script src="/layui/layui.all.js" type="text/javascript"></script>
|
||||
<script src="/javascript/header.js" type="text/javascript"></script>
|
||||
<script src="/javascript/user.js" type="text/javascript"></script>
|
||||
<script language="javascript" type="text/javascript">
|
||||
$(function () {
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/crawl/listCrawlByPage",
|
||||
data: {'curr':1,'limit':100},
|
||||
dataType: "json",
|
||||
success: function (data) {
|
||||
if (data.code == 200) {
|
||||
var crawlSourceList = data.data.list;
|
||||
if (crawlSourceList.length > 0) {
|
||||
var crawlSourceListHtml = "";
|
||||
for(var i=0;i<crawlSourceList.length;i++){
|
||||
var crawlSource = crawlSourceList[i];
|
||||
crawlSourceListHtml+=("<option value=\""+crawlSource.id+"\">"+crawlSource.sourceName+"</option>");
|
||||
}
|
||||
$("#sourceId").html(crawlSourceListHtml);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
} else if (data.code == 1001) {
|
||||
//未登录
|
||||
location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href);
|
||||
|
||||
}else {
|
||||
layer.alert(data.msg);
|
||||
}
|
||||
|
||||
},
|
||||
error: function () {
|
||||
layer.alert('网络异常');
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
|
||||
function addCrawlSingleTask() {
|
||||
|
||||
var sourceId = $("#sourceId").find("option:selected").val();
|
||||
|
||||
var catId = $("#catId").find("option:selected").val();
|
||||
|
||||
var sourceName = $("#sourceId").find("option:selected").html();
|
||||
|
||||
|
||||
var sourceBookId = $("#sourceBookId").val();
|
||||
|
||||
if (!sourceBookId){
|
||||
layer.alert("采集的源站小说ID不能为空");
|
||||
return;
|
||||
}
|
||||
|
||||
var bookName = $("#bookName").val();
|
||||
|
||||
if (!bookName){
|
||||
layer.alert("采集的小说名不能为空");
|
||||
return;
|
||||
}
|
||||
|
||||
var authorName = $("#authorName").val();
|
||||
|
||||
if (!authorName){
|
||||
layer.alert("采集的小说作者名不能为空");
|
||||
return;
|
||||
}
|
||||
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/crawl/addCrawlSingleTask",
|
||||
data: {'sourceId': sourceId, 'sourceName': sourceName,'catId':catId,'sourceBookId' : sourceBookId,'bookName':bookName,'authorName':authorName},
|
||||
dataType: "json",
|
||||
success: function (data) {
|
||||
if (data.code == 200) {
|
||||
|
||||
window.location.href = '/crawl/crawlSingleTask_list.html';
|
||||
|
||||
|
||||
} else {
|
||||
layer.alert(data.msg);
|
||||
}
|
||||
|
||||
},
|
||||
error: function () {
|
||||
layer.alert('网络异常');
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
</script>
|
||||
</html>
|
||||
@ -0,0 +1,230 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
||||
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
|
||||
<title>爬虫管理系统-小说精品屋</title>
|
||||
<link rel="stylesheet" href="/css/base.css?v=1"/>
|
||||
<link rel="stylesheet" href="/css/user.css"/>
|
||||
</head>
|
||||
</head>
|
||||
<body class="">
|
||||
|
||||
<div class="header">
|
||||
<div class="mainNav" id="mainNav">
|
||||
<div class="box_center cf"
|
||||
style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
|
||||
|
||||
小说精品屋爬虫管理
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="main box_center cf">
|
||||
<div class="userBox cf">
|
||||
<div class="my_l">
|
||||
|
||||
<ul class="log_list">
|
||||
<li><a class="link_1" href="/">爬虫源管理</a></li>
|
||||
<li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
|
||||
<!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
|
||||
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
<div class="my_r">
|
||||
<div class="my_bookshelf">
|
||||
<div class="title cf">
|
||||
<h2 class="fl">单本采集任务列表</h2>
|
||||
<div class="fr"><a href="/crawl/crawlSingleTask_add.html" class="btn_red">增加单本采集任务</a></div>
|
||||
</div>
|
||||
|
||||
<div id="divData" class="updateTable">
|
||||
<table cellpadding="0" cellspacing="0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="style">
|
||||
序号
|
||||
</th>
|
||||
<th class="name">
|
||||
采集小说名
|
||||
</th>
|
||||
<th class="name">
|
||||
采集小说作者名
|
||||
</th>
|
||||
<th class="goread">
|
||||
采集次数
|
||||
</th>
|
||||
<th class="goread">
|
||||
状态
|
||||
</th>
|
||||
<th class="name">
|
||||
创建时间
|
||||
</th>
|
||||
<th class="goread">
|
||||
操作
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="crawlSourceList">
|
||||
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
<div class="pageBox cf" id="shellPage">
|
||||
</div>
|
||||
</div>
|
||||
<!--<div id="divData" class="updateTable">
|
||||
<table cellpadding="0" cellspacing="0">
|
||||
<thead>
|
||||
<tr>
|
||||
|
||||
<th class="name">
|
||||
爬虫源（已开启的爬虫源）
|
||||
</th>
|
||||
<th class="chapter">
|
||||
成功爬取数量（websocket实现）
|
||||
</th>
|
||||
<th class="time">
|
||||
目标爬取数量
|
||||
</th>
|
||||
<th class="goread">
|
||||
状态（正在运行，已停止）（一次只能运行一个爬虫源）
|
||||
</th>
|
||||
<th class="goread">
|
||||
操作（启动，停止）
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="bookShelfList">
|
||||
|
||||
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
<div class="pageBox cf" id="shellPage">
|
||||
</div>
|
||||
</div>-->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</body>
|
||||
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
|
||||
<script src="/layui/layui.all.js" type="text/javascript"></script>
|
||||
<script src="/javascript/header.js" type="text/javascript"></script>
|
||||
<script src="/javascript/user.js" type="text/javascript"></script>
|
||||
<script language="javascript" type="text/javascript">
|
||||
search(1, 10);
|
||||
|
||||
function search(curr, limit) {
|
||||
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/crawl/listCrawlSingleTaskByPage",
|
||||
data: {'curr': curr, 'limit': limit},
|
||||
dataType: "json",
|
||||
success: function (data) {
|
||||
if (data.code == 200) {
|
||||
var crawlSourceList = data.data.list;
|
||||
if (crawlSourceList.length > 0) {
|
||||
var crawlSourceListHtml = "";
|
||||
for (var i = 0; i < crawlSourceList.length; i++) {
|
||||
var crawlSource = crawlSourceList[i];
|
||||
crawlSourceListHtml += (" <tr class=\"book_list\" vals=\"291\">\n" +
|
||||
" <td class=\"style bookclass\">\n" +
|
||||
" [" + (i + 1) + "]\n" +
|
||||
" </td>\n" +
|
||||
" <td class=\"name\">\n" +
|
||||
" " + crawlSource.bookName + "\n" +
|
||||
" </td>\n" +
|
||||
" <td class=\"name\">\n" +
|
||||
" " + crawlSource.authorName + "\n" +
|
||||
" </td>\n" +
|
||||
" <td class=\"goread\">\n" +
|
||||
" " + crawlSource.excCount + "\n" +
|
||||
" </td>\n" +
|
||||
" <td class=\"goread\">\n" +
|
||||
" " + (crawlSource.taskStatus == 0 ? '采集失败' : (crawlSource.taskStatus == 1 ? '采集成功' : (crawlSource.excCount>0?'采集中':'排队中'))) + "\n" +
|
||||
" </td>\n" +
|
||||
" <td class=\"name\" valsc=\"291|2037554|1\">"
|
||||
+ crawlSource.createTime + "</td>\n" +
|
||||
|
||||
" <td class=\"goread\" id='opt" + crawlSource.id + "'><a href='javascript:del(" + crawlSource.id + ")'>删除 </a></td> </tr>");
|
||||
}
|
||||
$("#crawlSourceList").html(crawlSourceListHtml);
|
||||
|
||||
layui.use('laypage', function () {
|
||||
var laypage = layui.laypage;
|
||||
|
||||
//执行一个laypage实例
|
||||
laypage.render({
|
||||
elem: 'shellPage' //注意，这里的 test1 是 ID，不用加 # 号
|
||||
, count: data.data.total //数据总数，从服务端得到,
|
||||
, curr: data.data.pageNum
|
||||
, limit: data.data.pageSize
|
||||
, jump: function (obj, first) {
|
||||
|
||||
|
||||
//obj包含了当前分页的所有参数，比如：
|
||||
console.log(obj.curr); //得到当前页，以便向服务端请求对应页的数据。
|
||||
console.log(obj.limit); //得到每页显示的条数
|
||||
|
||||
|
||||
//首次不执行
|
||||
if (!first) {
|
||||
search(obj.curr, obj.limit);
|
||||
} else {
|
||||
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
layer.alert(data.msg);
|
||||
}
|
||||
|
||||
},
|
||||
error: function () {
|
||||
layer.alert('网络异常');
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
function del(id) {
|
||||
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "/crawl/delCrawlSingleTask",
|
||||
data: {'id': id},
|
||||
dataType: "json",
|
||||
success: function (data) {
|
||||
if (data.code == 200) {
|
||||
location.reload();
|
||||
|
||||
|
||||
} else {
|
||||
layer.alert(data.msg);
|
||||
}
|
||||
|
||||
},
|
||||
error: function () {
|
||||
layer.alert('网络异常');
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
}
|
||||
</script>
|
||||
</html>
|
||||
@ -29,6 +29,7 @@
|
||||
|
||||
<ul class="log_list">
|
||||
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
|
||||
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
|
||||
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
|
||||
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
|
||||
</ul>
|
||||
|
||||
@ -28,6 +28,7 @@
|
||||
|
||||
<ul class="log_list">
|
||||
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
|
||||
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
|
||||
<!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
|
||||
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
|
||||
</ul>
|
||||
|
||||
40
sql/20200615.sql
Normal file
40
sql/20200615.sql
Normal file
@ -0,0 +1,40 @@
|
||||
/*
|
||||
Navicat MySQL Data Transfer
|
||||
|
||||
Source Server : localhost
|
||||
Source Server Version : 50725
|
||||
Source Host : localhost:3306
|
||||
Source Database : novel_plus
|
||||
|
||||
Target Server Type : MYSQL
|
||||
Target Server Version : 50725
|
||||
File Encoding : 65001
|
||||
|
||||
Date: 2020-06-15 15:06:55
|
||||
*/
|
||||
|
||||
SET FOREIGN_KEY_CHECKS=0;
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for crawl_single_task
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `crawl_single_task`;
|
||||
CREATE TABLE `crawl_single_task` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`source_id` int(11) DEFAULT NULL COMMENT '爬虫源ID',
|
||||
`source_name` varchar(50) DEFAULT NULL COMMENT '爬虫源名',
|
||||
`source_book_id` varchar(255) DEFAULT NULL COMMENT '源站小说ID',
|
||||
`cat_id` int(11) DEFAULT NULL COMMENT '分类ID',
|
||||
`book_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说名',
|
||||
`author_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说作者名',
|
||||
`task_status` tinyint(1) DEFAULT '2' COMMENT '任务状态，0：失败，1：成功，2；未执行',
|
||||
`exc_count` tinyint(2) DEFAULT '0' COMMENT '已经执行次数，最多执行5次',
|
||||
`create_time` datetime DEFAULT NULL COMMENT '创建时间',
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='抓取单本小说任务表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Records of crawl_single_task
|
||||
-- ----------------------------
|
||||
INSERT INTO `crawl_single_task` VALUES ('6', '2', '百书斋', '1', '1', '1', '1', '0', '5', '2020-06-15 14:36:07');
|
||||
INSERT INTO `crawl_single_task` VALUES ('7', '5', '笔趣阁', '108_108291', '1', '衍天志之不朽仙', '白衣少年丶', '1', '1', '2020-06-15 14:46:08');
|
||||
@ -1809,4 +1809,26 @@ CREATE TABLE `book_content9` (
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
|
||||
|
||||
|
||||
DROP TABLE IF EXISTS `crawl_single_task`;
|
||||
CREATE TABLE `crawl_single_task` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
|
||||
`source_id` int(11) DEFAULT NULL COMMENT '爬虫源ID',
|
||||
`source_name` varchar(50) DEFAULT NULL COMMENT '爬虫源名',
|
||||
`source_book_id` varchar(255) DEFAULT NULL COMMENT '源站小说ID',
|
||||
`cat_id` int(11) DEFAULT NULL COMMENT '分类ID',
|
||||
`book_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说名',
|
||||
`author_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说作者名',
|
||||
`task_status` tinyint(1) DEFAULT '2' COMMENT '任务状态，0：失败，1：成功，2；未执行',
|
||||
`exc_count` tinyint(2) DEFAULT '0' COMMENT '已经执行次数，最多执行5次',
|
||||
`create_time` datetime DEFAULT NULL COMMENT '创建时间',
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='抓取单本小说任务表';
|
||||
|
||||
-- ----------------------------
|
||||
-- Records of crawl_single_task
|
||||
-- ----------------------------
|
||||
INSERT INTO `crawl_single_task` VALUES ('6', '2', '百书斋', '1', '1', '1', '1', '0', '5', '2020-06-15 14:36:07');
|
||||
INSERT INTO `crawl_single_task` VALUES ('7', '5', '笔趣阁', '108_108291', '1', '衍天志之不朽仙', '白衣少年丶', '1', '1', '2020-06-15 14:46:08');
|
||||
UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n \"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"7\",\n \"catId6\": \"6\",\n \"catId7\": \"8\"\n },\n \"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n \"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n \"totalPagePatten\": \"第\\\\d+/(\\\\d+)页\",\n \"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n \"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n \"authorNamePatten\": \"<p>作者：([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"statusPatten\": \"<p>状态：([^/]+)</p>\",\n \"bookStatusRule\": {\n \"连载中\": 0,\n \"完本\": 1\n },\n \"descStart\": \"<div class=\\\"intro_info\\\">\",\n \"descEnd\": \"最新章节推荐地址\",\n \"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n \"bookIndexStart\": \"<dt>《\",\n \"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n \"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n \"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n \"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;
|
||||
Loading…
x
Reference in New Issue
Block a user