Compare commits

...

19 Commits

Author SHA1 Message Date
70f04bd37c 小说封面图片上传 2020-08-10 07:12:14 +08:00
9d621edaec update readme 2020-07-03 09:31:57 +08:00
56c0a81c1b update readme 2020-07-03 08:57:38 +08:00
6b3b88c147 Merge branch 'release_v2.6.1' into release_v2.6.2 2020-07-02 23:17:14 +08:00
894ee67f56 优化mac端浏览器的识别问题 2020-07-02 23:15:21 +08:00
a88891b72f v2.6.2发布 2020-06-29 16:45:51 +08:00
5c7724c813 修改首页缓存时间 2020-06-29 16:43:10 +08:00
79c9f3e0f1 Merge branch 'release_v2.6.0' into release_v2.6.1 2020-06-25 10:49:15 +08:00
575142f9f3 修复部分maven版本admin模块build失败的问题 2020-06-25 10:46:27 +08:00
83c2460c83 v2.6.1发布 2020-06-22 11:06:52 +08:00
c7642ac2ef 修复搜索引擎模式下按更新时间筛选的bug 2020-06-22 11:01:23 +08:00
5af570a514 v2.6.0发布 2020-06-15 18:08:09 +08:00
0d96ff81b6 v2.6.0发布 2020-06-15 17:47:12 +08:00
7733cf1e3f 更新版本号 2020-06-15 15:13:37 +08:00
c9c714e71e 增加单本采集任务 2020-06-15 15:08:15 +08:00
3cbb6bf3fb 遇到错误章节直接中断改为遇到错误章节跳过,解决某些网站由于大量章节错误而导致的大量书籍未爬取完的情况 2020-06-15 10:06:35 +08:00
750e234f60 更新 2020-06-09 04:26:32 +08:00
c1d485c42b 更新 2020-06-09 04:19:10 +08:00
0e2e6229cd 增加新源,过滤简介中的特殊字符,提高爬虫兼容性 2020-06-08 16:54:58 +08:00
38 changed files with 1213 additions and 120 deletions

View File

@ -1,10 +1,10 @@
[![index]( https://i.loli.net/2020/06/01/qF1uBMtswTIPJKx.jpg )]( https://www.eve.bz/cart.php?gid=11 )
[![index]( https://s1.ax1x.com/2020/07/03/NOSA5q.jpg )]( https://cloud.tencent.com/act/cps/redirect?redirect=1052&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console )
# 小说精品屋-plus
#### 新项目小说精品屋-微服务版
基于小说精品屋-plus构建的Spring Cloud 微服务小说门户平台可用于学习和商用
基于小说精品屋-plus构建的Spring Cloud 微服务小说门户平台
Gitee仓库地址 https://gitee.com/xiongxyang/novel-cloud
@ -203,4 +203,4 @@ docker安装教程[点击前往](https://my.oschina.net/java2nb/blog/4271989)
精品小说屋所有相关项目均已在开源中国公开感兴趣的可进入[开源中国](https://www.oschina.net/p/fiction_house)按关键字`精品小说屋`搜索。
[![index](./assets/120060.jpg)](https://www.aliyun.com/minisite/goods?userCode=uf4nasee )
[![index](https://s1.ax1x.com/2020/07/03/NOSuMF.jpg)](https://www.aliyun.com/minisite/goods?userCode=uf4nasee )

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.1 KiB

After

Width:  |  Height:  |  Size: 185 KiB

View File

@ -1,8 +0,0 @@
2020-05-13 21:52:00,972 INFO (StartupInfoLogger.java:50)- Starting TestDemo on USER-20180729KA with PID 3532 (started by Administrator in E:\baseprojectparent\novel-plus\novel-admin)
2020-05-13 21:52:01,113 DEBUG (StartupInfoLogger.java:53)- Running with Spring Boot v2.0.1.RELEASE, Spring v5.0.5.RELEASE
2020-05-13 21:52:01,131 INFO (SpringApplication.java:663)- The following profiles are active: dev
2020-05-13 21:52:54,469 DEBUG (ApplicationContextRegister.java:29)- ApplicationContext registed-->org.springframework.web.context.support.GenericWebApplicationContext@5b529706: startup date [Wed May 13 21:52:01 CST 2020]; root of context hierarchy
2020-05-13 21:53:49,622 INFO (StartupInfoLogger.java:59)- Started TestDemo in 114.268 seconds (JVM running for 124.957)
2020-05-18 09:48:03,219 INFO (StartupInfoLogger.java:50)- Starting TestDemo on DESKTOP-CPCLUI6 with PID 13172 (started by 11797 in D:\gitee\novel-plus\novel-admin)
2020-05-18 09:48:03,223 DEBUG (StartupInfoLogger.java:53)- Running with Spring Boot v2.0.1.RELEASE, Spring v5.0.5.RELEASE
2020-05-18 09:48:03,227 INFO (SpringApplication.java:663)- The following profiles are active: dev

View File

@ -137,10 +137,6 @@
<artifactId>velocity</artifactId>
<version>1.7</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<!--<dependency>-->
<!--<groupId>org.springframework.boot</groupId>-->
<!--<artifactId>spring-boot-starter-data-redis</artifactId>-->

View File

@ -12,7 +12,7 @@ spring:
driverClassName: com.mysql.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai
username: root
password:
password: test123456
#password:
initialSize: 1
minIdle: 3

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>2.5.0</version>
<version>2.6.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -55,19 +55,25 @@ public enum ResponseStatus {
* */
INVITE_CODE_INVALID(4001, "邀请码无效!"),
AUTHOR_STATUS_FORBIDDEN(4002, "作者状态异常,暂不能管理小说!")
, BOOKNAME_EXISTS(4003,"已发布过同名小说!")
, BOOKNAME_EXISTS(4003,"已发布过同名小说!"),
,
/**
* 小说相关错误
*/
BOOK_EXISTS(5001,"该小说已存在")
,
/**
* 搜索引擎相关错误
* */
ES_SEARCH_FAIL(9001,"密码错误!"),
ES_SEARCH_FAIL(9001,"搜索引擎查询错误!"),
/**
* 其他通用错误
* */
PASSWORD_ERROR(88001,"密码错误!");
private int code;
private String msg;

View File

@ -10,9 +10,21 @@ public class CrawlSingleTask {
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private Integer sourceId;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private String sourceName;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private String sourceBookId;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private Integer catId;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private String bookName;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private String authorName;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private Byte taskStatus;
@ -42,6 +54,16 @@ public class CrawlSingleTask {
this.sourceId = sourceId;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public String getSourceName() {
return sourceName;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public void setSourceName(String sourceName) {
this.sourceName = sourceName == null ? null : sourceName.trim();
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public String getSourceBookId() {
return sourceBookId;
@ -52,6 +74,36 @@ public class CrawlSingleTask {
this.sourceBookId = sourceBookId == null ? null : sourceBookId.trim();
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public Integer getCatId() {
return catId;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public void setCatId(Integer catId) {
this.catId = catId;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public String getBookName() {
return bookName;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public void setBookName(String bookName) {
this.bookName = bookName == null ? null : bookName.trim();
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public String getAuthorName() {
return authorName;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public void setAuthorName(String authorName) {
this.authorName = authorName == null ? null : authorName.trim();
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public Byte getTaskStatus() {
return taskStatus;

View File

@ -16,9 +16,21 @@ public final class CrawlSingleTaskDynamicSqlSupport {
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<Integer> sourceId = crawlSingleTask.sourceId;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<String> sourceName = crawlSingleTask.sourceName;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<String> sourceBookId = crawlSingleTask.sourceBookId;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<Integer> catId = crawlSingleTask.catId;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<String> bookName = crawlSingleTask.bookName;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<String> authorName = crawlSingleTask.authorName;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<Byte> taskStatus = crawlSingleTask.taskStatus;
@ -34,8 +46,16 @@ public final class CrawlSingleTaskDynamicSqlSupport {
public final SqlColumn<Integer> sourceId = column("source_id", JDBCType.INTEGER);
public final SqlColumn<String> sourceName = column("source_name", JDBCType.VARCHAR);
public final SqlColumn<String> sourceBookId = column("source_book_id", JDBCType.VARCHAR);
public final SqlColumn<Integer> catId = column("cat_id", JDBCType.INTEGER);
public final SqlColumn<String> bookName = column("book_name", JDBCType.VARCHAR);
public final SqlColumn<String> authorName = column("author_name", JDBCType.VARCHAR);
public final SqlColumn<Byte> taskStatus = column("task_status", JDBCType.TINYINT);
public final SqlColumn<Byte> excCount = column("exc_count", JDBCType.TINYINT);

View File

@ -35,7 +35,7 @@ import org.mybatis.dynamic.sql.util.mybatis3.MyBatis3Utils;
@Mapper
public interface CrawlSingleTaskMapper {
@Generated("org.mybatis.generator.api.MyBatisGenerator")
BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceBookId, taskStatus, excCount, createTime);
BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceName, sourceBookId, catId, bookName, authorName, taskStatus, excCount, createTime);
@Generated("org.mybatis.generator.api.MyBatisGenerator")
@SelectProvider(type=SqlProviderAdapter.class, method="select")
@ -63,7 +63,11 @@ public interface CrawlSingleTaskMapper {
@Results(id="CrawlSingleTaskResult", value = {
@Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
@Result(column="source_id", property="sourceId", jdbcType=JdbcType.INTEGER),
@Result(column="source_name", property="sourceName", jdbcType=JdbcType.VARCHAR),
@Result(column="source_book_id", property="sourceBookId", jdbcType=JdbcType.VARCHAR),
@Result(column="cat_id", property="catId", jdbcType=JdbcType.INTEGER),
@Result(column="book_name", property="bookName", jdbcType=JdbcType.VARCHAR),
@Result(column="author_name", property="authorName", jdbcType=JdbcType.VARCHAR),
@Result(column="task_status", property="taskStatus", jdbcType=JdbcType.TINYINT),
@Result(column="exc_count", property="excCount", jdbcType=JdbcType.TINYINT),
@Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP)
@ -96,7 +100,11 @@ public interface CrawlSingleTaskMapper {
return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c ->
c.map(id).toProperty("id")
.map(sourceId).toProperty("sourceId")
.map(sourceName).toProperty("sourceName")
.map(sourceBookId).toProperty("sourceBookId")
.map(catId).toProperty("catId")
.map(bookName).toProperty("bookName")
.map(authorName).toProperty("authorName")
.map(taskStatus).toProperty("taskStatus")
.map(excCount).toProperty("excCount")
.map(createTime).toProperty("createTime")
@ -108,7 +116,11 @@ public interface CrawlSingleTaskMapper {
return MyBatis3Utils.insertMultiple(this::insertMultiple, records, crawlSingleTask, c ->
c.map(id).toProperty("id")
.map(sourceId).toProperty("sourceId")
.map(sourceName).toProperty("sourceName")
.map(sourceBookId).toProperty("sourceBookId")
.map(catId).toProperty("catId")
.map(bookName).toProperty("bookName")
.map(authorName).toProperty("authorName")
.map(taskStatus).toProperty("taskStatus")
.map(excCount).toProperty("excCount")
.map(createTime).toProperty("createTime")
@ -120,7 +132,11 @@ public interface CrawlSingleTaskMapper {
return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c ->
c.map(id).toPropertyWhenPresent("id", record::getId)
.map(sourceId).toPropertyWhenPresent("sourceId", record::getSourceId)
.map(sourceName).toPropertyWhenPresent("sourceName", record::getSourceName)
.map(sourceBookId).toPropertyWhenPresent("sourceBookId", record::getSourceBookId)
.map(catId).toPropertyWhenPresent("catId", record::getCatId)
.map(bookName).toPropertyWhenPresent("bookName", record::getBookName)
.map(authorName).toPropertyWhenPresent("authorName", record::getAuthorName)
.map(taskStatus).toPropertyWhenPresent("taskStatus", record::getTaskStatus)
.map(excCount).toPropertyWhenPresent("excCount", record::getExcCount)
.map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime)
@ -158,7 +174,11 @@ public interface CrawlSingleTaskMapper {
static UpdateDSL<UpdateModel> updateAllColumns(CrawlSingleTask record, UpdateDSL<UpdateModel> dsl) {
return dsl.set(id).equalTo(record::getId)
.set(sourceId).equalTo(record::getSourceId)
.set(sourceName).equalTo(record::getSourceName)
.set(sourceBookId).equalTo(record::getSourceBookId)
.set(catId).equalTo(record::getCatId)
.set(bookName).equalTo(record::getBookName)
.set(authorName).equalTo(record::getAuthorName)
.set(taskStatus).equalTo(record::getTaskStatus)
.set(excCount).equalTo(record::getExcCount)
.set(createTime).equalTo(record::getCreateTime);
@ -168,7 +188,11 @@ public interface CrawlSingleTaskMapper {
static UpdateDSL<UpdateModel> updateSelectiveColumns(CrawlSingleTask record, UpdateDSL<UpdateModel> dsl) {
return dsl.set(id).equalToWhenPresent(record::getId)
.set(sourceId).equalToWhenPresent(record::getSourceId)
.set(sourceName).equalToWhenPresent(record::getSourceName)
.set(sourceBookId).equalToWhenPresent(record::getSourceBookId)
.set(catId).equalToWhenPresent(record::getCatId)
.set(bookName).equalToWhenPresent(record::getBookName)
.set(authorName).equalToWhenPresent(record::getAuthorName)
.set(taskStatus).equalToWhenPresent(record::getTaskStatus)
.set(excCount).equalToWhenPresent(record::getExcCount)
.set(createTime).equalToWhenPresent(record::getCreateTime);
@ -178,7 +202,11 @@ public interface CrawlSingleTaskMapper {
default int updateByPrimaryKey(CrawlSingleTask record) {
return update(c ->
c.set(sourceId).equalTo(record::getSourceId)
.set(sourceName).equalTo(record::getSourceName)
.set(sourceBookId).equalTo(record::getSourceBookId)
.set(catId).equalTo(record::getCatId)
.set(bookName).equalTo(record::getBookName)
.set(authorName).equalTo(record::getAuthorName)
.set(taskStatus).equalTo(record::getTaskStatus)
.set(excCount).equalTo(record::getExcCount)
.set(createTime).equalTo(record::getCreateTime)
@ -190,7 +218,11 @@ public interface CrawlSingleTaskMapper {
default int updateByPrimaryKeySelective(CrawlSingleTask record) {
return update(c ->
c.set(sourceId).equalToWhenPresent(record::getSourceId)
.set(sourceName).equalToWhenPresent(record::getSourceName)
.set(sourceBookId).equalToWhenPresent(record::getSourceBookId)
.set(catId).equalToWhenPresent(record::getCatId)
.set(bookName).equalToWhenPresent(record::getBookName)
.set(authorName).equalToWhenPresent(record::getAuthorName)
.set(taskStatus).equalToWhenPresent(record::getTaskStatus)
.set(excCount).equalToWhenPresent(record::getExcCount)
.set(createTime).equalToWhenPresent(record::getCreateTime)

View File

@ -42,12 +42,12 @@ sharding:
jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: test123456
# ds1:
# type: com.alibaba.druid.pool.DruidDataSource
# driver-class-name: com.mysql.jdbc.Driver
# url: jdbc:mysql://localhost:3306/novel_plus2
# username: root
# password: test123456
# ds1:
# type: com.alibaba.druid.pool.DruidDataSource
# driver-class-name: com.mysql.jdbc.Driver
# url: jdbc:mysql://localhost:3306/novel_plus2
# username: root
# password: test123456
config:
sharding:
props:
@ -56,10 +56,10 @@ sharding:
book_content: #book_content表
key-generator-column-name: id #主键
actual-data-nodes: ds${0}.book_content${0..9} #数据节点
# database-strategy: #分库策略
# inline:
# sharding-column: book_id
# algorithm-expression: ds${book_id % 10}
# database-strategy: #分库策略
# inline:
# sharding-column: book_id
# algorithm-expression: ds${book_id % 10}
table-strategy: #分表策略
inline:
shardingColumn: index_id
@ -74,8 +74,4 @@ pic:
save:
type: 2 #图片保存方式, 1不保存使用爬取的网络图片 2保存在自己的存储介质
storage: local #存储介质local本地OSS阿里云对象存储fastDfs分布式文件系统
path: /var/pic #图片保存路径
path: /var/pic #图片保存路径

View File

@ -9,7 +9,7 @@
</commentGenerator>
<jdbcConnection
connectionURL="jdbc:mysql://127.0.0.1:3306/novel_plus?tinyInt1isBit=false&amp;useUnicode=true&amp;characterEncoding=utf-8&amp;serverTimezone=Asia/Shanghai&amp;nullCatalogMeansCurrent=true"
driverClass="com.mysql.jdbc.Driver" password=""
driverClass="com.mysql.jdbc.Driver" password="test123456"
userId="root" />
<!-- 默认false把JDBC DECIMAL NUMERIC 类型解析为 Integer true时把JDBC DECIMAL
@ -44,7 +44,7 @@
</javaClientGenerator>
<!--生成全部表tableName设为%-->
<table tableName="user_buy_record" domainObjectName="UserBuyRecord"/>
<table tableName="crawl_single_task"/>
<!-- 指定数据库表 -->
<!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>2.5.0</version>
<version>2.6.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,11 +3,12 @@ package com.java2nb.novel.controller;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.core.bean.ResultBean;
import com.java2nb.novel.core.utils.BeanUtil;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.vo.CrawlSingleTaskVO;
import com.java2nb.novel.vo.CrawlSourceVO;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@ -56,6 +57,38 @@ public class CrawlController {
return ResultBean.ok();
}
/**
* 新增单本采集任务
* */
@PostMapping("addCrawlSingleTask")
public ResultBean addCrawlSingleTask(CrawlSingleTask singleTask){
crawlService.addCrawlSingleTask(singleTask);
return ResultBean.ok();
}
/**
* 单本采集任务分页列表查询
* */
@PostMapping("listCrawlSingleTaskByPage")
public ResultBean listCrawlSingleTaskByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize){
return ResultBean.ok(new PageInfo<>(BeanUtil.copyList(crawlService.listCrawlSingleTaskByPage(page,pageSize), CrawlSingleTaskVO.class)
));
}
/**
* 删除采集任务
* */
@PostMapping("delCrawlSingleTask")
public ResultBean delCrawlSingleTask(Long id){
crawlService.delCrawlSingleTask(id);
return ResultBean.ok();
}

View File

@ -96,8 +96,12 @@ public class CrawlParser {
String desc = bookDetailHtml.substring(bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length());
desc = desc.substring(0, desc.indexOf(ruleBean.getDescEnd()));
//过滤掉简介中的a标签
desc = desc.replaceAll("<a[^<]+</a>","");
//过滤掉简介中的特殊标签
desc = desc.replaceAll("<a[^<]+</a>","")
.replaceAll("<font[^<]+</font>","")
.replaceAll("<p>\\s*</p>","")
.replaceAll("<p>","")
.replaceAll("</p>","<br/>");
//设置书籍简介
book.setBookDesc(desc);
if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {
@ -237,8 +241,6 @@ public class CrawlParser {
}
} else {
break;
}

View File

@ -3,10 +3,7 @@ package com.java2nb.novel.core.listener;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.java2nb.novel.core.crawl.CrawlParser;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.entity.*;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.utils.Constants;
@ -40,15 +37,15 @@ public class StarterListener implements ServletContextListener {
@Override
public void contextInitialized(ServletContextEvent sce) {
log.info("程序启动,开始执行自动更新线程。。。");
for(int i = 0 ; i<updateThreadCount; i++) {
for (int i = 0; i < updateThreadCount; i++) {
new Thread(() -> {
log.info("程序启动,开始执行自动更新线程。。。");
while (true) {
try {
//1.查询最新目录更新时间在一个月之内的前100条需要更新的数据
Date currentDate = new Date();
Date startDate = DateUtils.addDays(currentDate, -30);
List<Book> bookList ;
List<Book> bookList;
synchronized (this) {
bookList = bookService.queryNeedUpdateBook(startDate, 100);
}
@ -61,7 +58,7 @@ public class StarterListener implements ServletContextListener {
Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
//这里只做老书更新
book.setId(needUpdateBook.getId());
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
if (needUpdateBook.getPicUrl() != null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
//本地图片则不更新
book.setPicUrl(null);
}
@ -83,6 +80,42 @@ public class StarterListener implements ServletContextListener {
}
}).start();
}
new Thread(() -> {
log.info("程序启动,开始执行单本采集任务线程。。。");
while (true) {
CrawlSingleTask task = null;
byte crawlStatus = 0;
try {
//获取采集任务
task = crawlService.getCrawlSingleTask();
if (task != null) {
//查询爬虫规则
CrawlSource source = crawlService.queryCrawlSource(task.getSourceId());
RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
if (crawlService.parseBookAndSave(task.getCatId(), ruleBean, task.getSourceId(), task.getSourceBookId())) {
//采集成功
crawlStatus = 1;
}
}
Thread.sleep(1000 * 60);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
if (task != null) {
crawlService.updateCrawlSingleTask(task, crawlStatus);
}
}
}).start();
}
}

View File

@ -1,6 +1,7 @@
package com.java2nb.novel.service;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import java.util.List;
@ -39,6 +40,16 @@ public interface CrawlService {
* */
void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus);
/**
* 采集并保存小说
* @param catId 分类ID
* @param bookId 小说ID
* @param sourceId 源ID
* @param ruleBean 采集规则\
* @return true:成功false:失败
* */
boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId);
/**
* 根据爬虫状态查询爬虫源集合
* @param sourceStatus 状态0关闭1开启
@ -61,4 +72,37 @@ public interface CrawlService {
* @return 源信息
* */
CrawlSource queryCrawlSource(Integer sourceId);
/**
* 新增单本采集任务
* @param singleTask 任务信息对象
* */
void addCrawlSingleTask(CrawlSingleTask singleTask);
/**
* 单本采集任务分页列表查询
* @param page 当前页码
* @param pageSize 分页大小
* @return 单本采集任务集合
* */
List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize);
/**
* 删除采集任务
* @param id 任务ID
* */
void delCrawlSingleTask(Long id);
/**
* 获取采集任务
* @return 采集任务
* */
CrawlSingleTask getCrawlSingleTask();
/**
* 更新单本采集任务
* @param task 采集任务
* @param status 采集状态
* */
void updateCrawlSingleTask(CrawlSingleTask task, Byte status);
}

View File

@ -6,12 +6,12 @@ import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.crawl.CrawlParser;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.core.enums.ResponseStatus;
import com.java2nb.novel.core.exception.BusinessException;
import com.java2nb.novel.core.utils.IdWorker;
import com.java2nb.novel.core.utils.SpringUtil;
import com.java2nb.novel.core.utils.ThreadUtil;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.entity.*;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.mapper.*;
import com.java2nb.novel.service.BookService;
@ -33,8 +33,7 @@ import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClient;
import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlBookId;
import static com.java2nb.novel.mapper.BookDynamicSqlSupport.crawlSourceId;
import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*;
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.update;
import static org.mybatis.dynamic.sql.SqlBuilder.*;
import static org.mybatis.dynamic.sql.select.SelectDSL.select;
/**
@ -48,6 +47,8 @@ public class CrawlServiceImpl implements CrawlService {
private final CrawlSourceMapper crawlSourceMapper;
private final CrawlSingleTaskMapper crawlSingleTaskMapper;
private final BookService bookService;
@ -140,6 +141,62 @@ public class CrawlServiceImpl implements CrawlService {
return crawlSourceMapper.selectMany(render).get(0);
}
@Override
public void addCrawlSingleTask(CrawlSingleTask singleTask) {
if(bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(),singleTask.getAuthorName())){
throw new BusinessException(ResponseStatus.BOOK_EXISTS);
}
singleTask.setCreateTime(new Date());
crawlSingleTaskMapper.insertSelective(singleTask);
}
@Override
public List<CrawlSingleTask> listCrawlSingleTaskByPage(int page, int pageSize) {
PageHelper.startPage(page, pageSize);
SelectStatementProvider render = select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
.from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime.descending())
.build()
.render(RenderingStrategies.MYBATIS3);
return crawlSingleTaskMapper.selectMany(render);
}
@Override
public void delCrawlSingleTask(Long id) {
crawlSingleTaskMapper.deleteByPrimaryKey(id);
}
@Override
public CrawlSingleTask getCrawlSingleTask() {
List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
.from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
.where(CrawlSingleTaskDynamicSqlSupport.taskStatus,isEqualTo((byte)2))
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime)
.limit(1)
.build()
.render(RenderingStrategies.MYBATIS3));
return list.size() > 0 ? list.get(0) : null;
}
@Override
public void updateCrawlSingleTask(CrawlSingleTask task, Byte status) {
byte excCount = task.getExcCount();
excCount+=1;
task.setExcCount(excCount);
if(status == 1 || excCount == 5){
//当采集成功或者采集次数等于5则更新采集最终状态并停止采集
task.setTaskStatus(status);
}
crawlSingleTaskMapper.updateByPrimaryKeySelective(task);
}
/**
* 解析分类列表
*/
@ -173,35 +230,7 @@ public class CrawlServiceImpl implements CrawlService {
String bookId = bookIdMatcher.group(1);
Book book = CrawlParser.parseBook(ruleBean, bookId);
//这里只做新书入库,查询是否存在这本书
Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
//如果该小说不存在则可以解析入库但是标记该小说正在入库30分钟之后才允许再次入库
if (existBook == null) {
//没有该书,可以入库
book.setCatId(catId);
//根据分类ID查询分类
book.setCatName(bookService.queryCatNameByCatId(catId));
if (catId == 7) {
//女频
book.setWorkDirection((byte) 1);
} else {
//男频
book.setWorkDirection((byte) 0);
}
book.setCrawlBookId(bookId);
book.setCrawlSourceId(sourceId);
book.setCrawlLastTime(new Date());
book.setId(new IdWorker().nextId());
//解析章节目录
Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
} else {
//只更新书籍的爬虫相关字段
bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
}
parseBookAndSave(catId, ruleBean, sourceId, bookId);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
@ -232,6 +261,43 @@ public class CrawlServiceImpl implements CrawlService {
}
@Override
public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) {
Book book = CrawlParser.parseBook(ruleBean, bookId);
if(book.getBookName() == null || book.getAuthorName() == null){
return false;
}
//这里只做新书入库,查询是否存在这本书
Book existBook = bookService.queryBookByBookNameAndAuthorName(book.getBookName(), book.getAuthorName());
//如果该小说不存在则可以解析入库但是标记该小说正在入库30分钟之后才允许再次入库
if (existBook == null) {
//没有该书,可以入库
book.setCatId(catId);
//根据分类ID查询分类
book.setCatName(bookService.queryCatNameByCatId(catId));
if (catId == 7) {
//女频
book.setWorkDirection((byte) 1);
} else {
//男频
book.setWorkDirection((byte) 0);
}
book.setCrawlBookId(bookId);
book.setCrawlSourceId(sourceId);
book.setCrawlLastTime(new Date());
book.setId(new IdWorker().nextId());
//解析章节目录
Map<Integer, List> indexAndContentList = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0));
bookService.saveBookAndIndexAndContent(book, (List<BookIndex>) indexAndContentList.get(CrawlParser.BOOK_INDEX_LIST_KEY), (List<BookContent>) indexAndContentList.get(CrawlParser.BOOK_CONTENT_LIST_KEY));
} else {
//只更新书籍的爬虫相关字段
bookService.updateCrawlProperties(existBook.getId(), sourceId, bookId);
}
return true;
}
@Override
public void updateCrawlSourceStatus(Integer sourceId, Byte sourceStatus) {
CrawlSource source = new CrawlSource();

View File

@ -0,0 +1,26 @@
package com.java2nb.novel.vo;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import lombok.Data;
import java.util.Date;
/**
* @author Administrator
*/
@Data
public class CrawlSingleTaskVO extends CrawlSingleTask {
@JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm")
private Date createTime;
@Override
public String toString() {
return super.toString();
}
}

View File

@ -10,7 +10,7 @@
.user_l .log_list { width:350px }
.user_l .s_input { margin-bottom:25px; font-size:14px }
.s_input { width:348px; height:30px; line-height:38px\9; vertical-align:middle; border:1px solid #ddd; border-radius:2px }
.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px; background:url(../images/icon_user.png) no-repeat 13px 13px }
.icon_name, .icon_key, .icon_code { width:312px; padding-left:36px}
.icon_key { background-position: 13px -51px }
.icon_code { background-position: 13px -117px; width:200px; float:left }
.code_pic { height:38px; float:right }
@ -37,7 +37,7 @@
.fast_tit .title { background:#fff; font-size:16px; padding:3px 14px; position:relative; display:inline-block; z-index:999 }
/*userinfo*/
.my_l { width:198px; float:left; font-size: 13px; padding-top: 20px; }
.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff; background:url(../images/icon_user.png) no-repeat; margin-bottom:5px; color: #666 }
.my_l li a { display:block; height:42px; line-height:42px; padding-left:62px; border-left:4px solid #fff; margin-bottom:5px; color: #666 }
.my_l li .on { background-color:#fafafa; border-left:2px solid #f80; color:#000; border-radius: 0 2px 2px 0 }
.my_l .link_1 { background-position:32px -188px }
.my_l .link_2 { background-position:32px -230px }

View File

@ -0,0 +1,188 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
<title>爬虫管理系统-小说精品屋</title>
<link rel="stylesheet" href="/css/base.css?v=1"/>
<link rel="stylesheet" href="/css/user.css"/>
</head>
</head>
<body class="">
<div class="header">
<div class="mainNav" id="mainNav">
<div class="box_center cf"
style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
小说精品屋爬虫管理
</div>
</div>
</div>
<div class="main box_center cf">
<div class="userBox cf">
<div class="my_l">
<ul class="log_list">
<li><a class="link_1" href="/">爬虫源管理</a></li>
<li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>
</div>
<div class="my_r">
<div class="my_bookshelf">
<div class="userBox cf">
<form method="post" action="./register.html" id="form2">
<div class="user_l">
<div></div>
<h3>采集信息填写示例均为笔趣阁http://www.mcmssc.com</h3>
<ul class="log_list">
<li><span id="LabErr"></span></li>
<input type="hidden" id="sourceName" class="s_input icon_name" >
<b>采集源:</b>
<li><select id="sourceId" class="s_input icon_key">
</select></li>
<b>采集分类:</b>
<li><select id="catId" class="s_input icon_key">
<option value="1">玄幻奇幻</option>
<option value="2">武侠仙侠</option>
<option value="3">都市言情</option>
<option value="4">历史军事</option>
<option value="5">科幻灵异</option>
<option value="6">网游竞技</option>
<option value="7">女生频道</option>
</select>
</li>
示例:<b>73_73911</b>
<li><input type="text" id="sourceBookId" class="s_input icon_key"
placeholder="采集的源站小说ID"/></li>
示例:<b>苏厨</b>
<li><input type="text" id="bookName" class="s_input icon_key"
placeholder="采集的小说名"/></li>
示例:<b>二子从周</b>
<li><input type="text" id="authorName" class="s_input icon_key"
placeholder="采集的小说作者名"/></li>
<!--示例:<b>https://m.xdingdiann.com/sort/0/1.html</b>
<li><input type="text" id="updateBookListUrl" class="s_input icon_key"
placeholder="小说更新列表url"></li>-->
<li><input type="button" onclick="addCrawlSingleTask()" name="btnRegister" value="提交"
id="btnRegister" class="btn_red"></li>
</ul>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
<script src="/layui/layui.all.js" type="text/javascript"></script>
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
$(function () {
$.ajax({
type: "POST",
url: "/crawl/listCrawlByPage",
data: {'curr':1,'limit':100},
dataType: "json",
success: function (data) {
if (data.code == 200) {
var crawlSourceList = data.data.list;
if (crawlSourceList.length > 0) {
var crawlSourceListHtml = "";
for(var i=0;i<crawlSourceList.length;i++){
var crawlSource = crawlSourceList[i];
crawlSourceListHtml+=("<option value=\""+crawlSource.id+"\">"+crawlSource.sourceName+"</option>");
}
$("#sourceId").html(crawlSourceListHtml);
}
} else if (data.code == 1001) {
//未登录
location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href);
}else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
});
function addCrawlSingleTask() {
var sourceId = $("#sourceId").find("option:selected").val();
var catId = $("#catId").find("option:selected").val();
var sourceName = $("#sourceId").find("option:selected").html();
var sourceBookId = $("#sourceBookId").val();
if (!sourceBookId){
layer.alert("采集的源站小说ID不能为空");
return;
}
var bookName = $("#bookName").val();
if (!bookName){
layer.alert("采集的小说名不能为空");
return;
}
var authorName = $("#authorName").val();
if (!authorName){
layer.alert("采集的小说作者名不能为空");
return;
}
$.ajax({
type: "POST",
url: "/crawl/addCrawlSingleTask",
data: {'sourceId': sourceId, 'sourceName': sourceName,'catId':catId,'sourceBookId' : sourceBookId,'bookName':bookName,'authorName':authorName},
dataType: "json",
success: function (data) {
if (data.code == 200) {
window.location.href = '/crawl/crawlSingleTask_list.html';
} else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
</script>
</html>

View File

@ -0,0 +1,230 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
<title>爬虫管理系统-小说精品屋</title>
<link rel="stylesheet" href="/css/base.css?v=1"/>
<link rel="stylesheet" href="/css/user.css"/>
</head>
</head>
<body class="">
<div class="header">
<div class="mainNav" id="mainNav">
<div class="box_center cf"
style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
小说精品屋爬虫管理
</div>
</div>
</div>
<div class="main box_center cf">
<div class="userBox cf">
<div class="my_l">
<ul class="log_list">
<li><a class="link_1" href="/">爬虫源管理</a></li>
<li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>
</div>
<div class="my_r">
<div class="my_bookshelf">
<div class="title cf">
<h2 class="fl">单本采集任务列表</h2>
<div class="fr"><a href="/crawl/crawlSingleTask_add.html" class="btn_red">增加单本采集任务</a></div>
</div>
<div id="divData" class="updateTable">
<table cellpadding="0" cellspacing="0">
<thead>
<tr>
<th class="style">
序号
</th>
<th class="name">
采集小说名
</th>
<th class="name">
采集小说作者名
</th>
<th class="goread">
采集次数
</th>
<th class="goread">
状态
</th>
<th class="name">
创建时间
</th>
<th class="goread">
操作
</th>
</tr>
</thead>
<tbody id="crawlSourceList">
</tbody>
</table>
<div class="pageBox cf" id="shellPage">
</div>
</div>
<!--<div id="divData" class="updateTable">
<table cellpadding="0" cellspacing="0">
<thead>
<tr>
<th class="name">
爬虫源(已开启的爬虫源)
</th>
<th class="chapter">
成功爬取数量websocket实现
</th>
<th class="time">
目标爬取数量
</th>
<th class="goread">
状态(正在运行,已停止)(一次只能运行一个爬虫源)
</th>
<th class="goread">
操作(启动,停止)
</th>
</tr>
</thead>
<tbody id="bookShelfList">
</tbody>
</table>
<div class="pageBox cf" id="shellPage">
</div>
</div>-->
</div>
</div>
</div>
</div>
</body>
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
<script src="/layui/layui.all.js" type="text/javascript"></script>
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
search(1, 10);
function search(curr, limit) {
$.ajax({
type: "POST",
url: "/crawl/listCrawlSingleTaskByPage",
data: {'curr': curr, 'limit': limit},
dataType: "json",
success: function (data) {
if (data.code == 200) {
var crawlSourceList = data.data.list;
if (crawlSourceList.length > 0) {
var crawlSourceListHtml = "";
for (var i = 0; i < crawlSourceList.length; i++) {
var crawlSource = crawlSourceList[i];
crawlSourceListHtml += (" <tr class=\"book_list\" vals=\"291\">\n" +
" <td class=\"style bookclass\">\n" +
" [" + (i + 1) + "]\n" +
" </td>\n" +
" <td class=\"name\">\n" +
" " + crawlSource.bookName + "\n" +
" </td>\n" +
" <td class=\"name\">\n" +
" " + crawlSource.authorName + "\n" +
" </td>\n" +
" <td class=\"goread\">\n" +
" " + crawlSource.excCount + "\n" +
" </td>\n" +
" <td class=\"goread\">\n" +
" " + (crawlSource.taskStatus == 0 ? '采集失败' : (crawlSource.taskStatus == 1 ? '采集成功' : (crawlSource.excCount>0?'采集中':'排队中'))) + "\n" +
" </td>\n" +
" <td class=\"name\" valsc=\"291|2037554|1\">"
+ crawlSource.createTime + "</td>\n" +
" <td class=\"goread\" id='opt" + crawlSource.id + "'><a href='javascript:del(" + crawlSource.id + ")'>删除 </a></td> </tr>");
}
$("#crawlSourceList").html(crawlSourceListHtml);
layui.use('laypage', function () {
var laypage = layui.laypage;
//执行一个laypage实例
laypage.render({
elem: 'shellPage' //注意,这里的 test1 是 ID不用加 # 号
, count: data.data.total //数据总数,从服务端得到,
, curr: data.data.pageNum
, limit: data.data.pageSize
, jump: function (obj, first) {
//obj包含了当前分页的所有参数比如
console.log(obj.curr); //得到当前页,以便向服务端请求对应页的数据。
console.log(obj.limit); //得到每页显示的条数
//首次不执行
if (!first) {
search(obj.curr, obj.limit);
} else {
}
}
});
});
}
} else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
function del(id) {
$.ajax({
type: "POST",
url: "/crawl/delCrawlSingleTask",
data: {'id': id},
dataType: "json",
success: function (data) {
if (data.code == 200) {
location.reload();
} else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
</script>
</html>

View File

@ -29,6 +29,7 @@
<ul class="log_list">
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>

View File

@ -28,6 +28,7 @@
<ul class="log_list">
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>2.5.0</version>
<version>2.6.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -1,17 +1,32 @@
package com.java2nb.novel.controller;
import com.java2nb.novel.core.bean.ResultBean;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.core.utils.RandomValidateCodeUtil;
import com.java2nb.novel.core.utils.RestTemplateUtil;
import com.java2nb.novel.core.utils.UUIDUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.Charsets;
import org.apache.http.client.utils.DateUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.imageio.ImageIO;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import java.io.*;
import java.util.Date;
import java.util.Objects;
/**
* @author 11797
@ -24,6 +39,9 @@ public class FileController {
private final CacheService cacheService;
@Value("${pic.save.path}")
private String picSavePath;
/**
* 生成验证码
*/
@ -44,5 +62,32 @@ public class FileController {
}
}
/**
* 文件上传
*/
@ResponseBody
@PostMapping("/upload")
ResultBean upload(@RequestParam("file") MultipartFile file) {
Date currentDate = new Date();
try {
String savePath =
Constants.LOCAL_PIC_PREFIX + DateUtils.formatDate(currentDate, "yyyy") + "/" +
DateUtils.formatDate(currentDate, "MM") + "/" +
DateUtils.formatDate(currentDate, "dd") ;
String oriName = file.getOriginalFilename();
String saveFileName = UUIDUtil.getUUID32() + oriName.substring(oriName.lastIndexOf("."));
File saveFile = new File( picSavePath + savePath, saveFileName);
if (!saveFile.getParentFile().exists()) {
saveFile.getParentFile().mkdirs();
}
file.transferTo(saveFile);
return ResultBean.ok(savePath+"/"+saveFileName);
} catch (Exception e) {
log.error(e.getMessage(), e);
return ResultBean.error();
}
}
}

View File

@ -18,19 +18,20 @@ public class BrowserUtil {
"lg-d", "lg-g", "lge-", "maui", "maxo", "midp", "mits", "mmef", "mobi", "mot-", "moto", "mwbp", "nec-",
"newt", "noki", "oper", "palm", "pana", "pant", "phil", "play", "port", "prox", "qwap", "sage", "sams",
"sany", "sch-", "sec-", "send", "seri", "sgh-", "shar", "sie-", "siem", "smal", "smar", "sony", "sph-",
"symb", "t-mo", "teli", "tim-", "tosh", "tsm-", "upg1", "upsi", "vk-v", "voda", "wap-", "wapa", "wapi",
"symb", "t-mo", "teli", "tim-", "tsm-", "upg1", "upsi", "vk-v", "voda", "wap-", "wapa", "wapi",
"wapp", "wapr", "webc", "winw", "winw", "xda", "xda-", "Googlebot-Mobile" };
/**
*
* @Title: JudgelsMobile @Description: TODO(判断是否是手机浏览器) @param @param
* request @param @return 设定文件 @return boolean 返回类型 @throws
* 判断是否是手机浏览器
*/
public static boolean isMobile(HttpServletRequest request) {
boolean isMobile = false;
if (request.getHeader("User-Agent") != null) {
String userAgent = request.getHeader("User-Agent");
if (userAgent != null) {
userAgent = userAgent.toLowerCase();
for (String mobileAgent : mobileAgents) {
if (request.getHeader("User-Agent").toLowerCase().indexOf(mobileAgent) > 0) {
if (userAgent.indexOf(mobileAgent) > 0) {
isMobile = true;
break;
}

View File

@ -40,7 +40,7 @@ public class FriendLinkServiceImpl implements FriendLinkService {
.build()
.render(RenderingStrategies.MYBATIS3);
result = friendLinkMapper.selectMany(selectStatement);
cacheService.setObject(CacheKey.INDEX_LINK_KEY,result);
cacheService.setObject(CacheKey.INDEX_LINK_KEY,result,60 * 60 * 24);
}
return result;
}

View File

@ -42,7 +42,7 @@ public class NewsServiceImpl implements NewsService {
.build()
.render(RenderingStrategies.MYBATIS3);
result = newsMapper.selectMany(selectStatement);
cacheService.setObject(CacheKey.INDEX_NEWS_KEY,result);
cacheService.setObject(CacheKey.INDEX_NEWS_KEY,result,60 * 60 * 12);
}
return result;
}

View File

@ -93,7 +93,7 @@ public class SearchServiceImpl implements SearchService {
boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
if (params.getUpdateTimeMin() != null) {
boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(params.getUpdateTimeMin()));
boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(params.getUpdateTimeMin())));
}
@ -134,8 +134,8 @@ public class SearchServiceImpl implements SearchService {
log.debug(search.toString());
SearchResult result;
result = jestClient.execute(search);
log.debug(result.getJsonString());
if (result.isSucceeded()) {
log.debug(result.getJsonString());
Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
if (resultMap.get("hits") != null) {
@ -196,4 +196,7 @@ public class SearchServiceImpl implements SearchService {
}
throw new BusinessException(ResponseStatus.ES_SEARCH_FAIL);
}
}

View File

@ -0,0 +1,217 @@
jQuery.extend({
createUploadIframe: function(id, uri)
{
//create frame
var frameId = 'jUploadFrame' + id;
var iframeHtml = '<iframe id="' + frameId + '" name="' + frameId + '" style="position:absolute; top:-9999px; left:-9999px"';
if(window.ActiveXObject)
{
if(typeof uri== 'boolean'){
iframeHtml += ' src="' + 'javascript:false' + '"';
}
else if(typeof uri== 'string'){
iframeHtml += ' src="' + uri + '"';
}
}
iframeHtml += ' />';
jQuery(iframeHtml).appendTo(document.body);
return jQuery('#' + frameId).get(0);
},
createUploadForm: function(id,fileElementId,data,fileElement)
{
//create form
var formId = 'jUploadForm' + id;
var fileId = 'jUploadFile' + id;
var form = jQuery('<form action="" method="POST" name="' + formId + '" id="' + formId + '" enctype="multipart/form-data"></form>');
if(data)
{
for(var i in data)
{
jQuery('<input type="hidden" name="' + i + '" value="' + data[i] + '" />').appendTo(form);
}
}
var oldElement;
if(fileElement == null)
oldElement = jQuery('#' + fileElementId);
else
oldElement = fileElement;
var newElement = jQuery(oldElement).clone();
jQuery(oldElement).attr('id', fileId);
jQuery(oldElement).before(newElement);
jQuery(oldElement).appendTo(form);
//set attributes
jQuery(form).css('position', 'absolute');
jQuery(form).css('top', '-1200px');
jQuery(form).css('left', '-1200px');
jQuery(form).appendTo('body');
return form;
},
ajaxFileUpload: function(s) {
// TODO introduce global settings, allowing the client to modify them for all requests, not only timeout
s = jQuery.extend({}, jQuery.ajaxSettings, s);
var id = new Date().getTime()
var form = jQuery.createUploadForm(id, s.fileElementId, (typeof(s.data)=='undefined'?false:s.data),s.fileElement);
var io = jQuery.createUploadIframe(id, s.secureuri);
var frameId = 'jUploadFrame' + id;
var formId = 'jUploadForm' + id;
// Watch for a new set of requests
if ( s.global && ! jQuery.active++ )
{
jQuery.event.trigger( "ajaxStart" );
}
var requestDone = false;
// Create the request object
var xml = {}
if ( s.global )
jQuery.event.trigger("ajaxSend", [xml, s]);
// Wait for a response to come back
var uploadCallback = function(isTimeout)
{
var io = document.getElementById(frameId);
try
{
if(io.contentWindow)
{
xml.responseText = io.contentWindow.document.body?io.contentWindow.document.body.innerHTML:null;
xml.responseXML = io.contentWindow.document.XMLDocument?io.contentWindow.document.XMLDocument:io.contentWindow.document;
}else if(io.contentDocument)
{
xml.responseText = io.contentDocument.document.body?io.contentDocument.document.body.innerHTML:null;
xml.responseXML = io.contentDocument.document.XMLDocument?io.contentDocument.document.XMLDocument:io.contentDocument.document;
}
}catch(e)
{
jQuery.handleError(s, xml, null, e);
}
if ( xml || isTimeout == "timeout")
{
requestDone = true;
var status;
try {
status = isTimeout != "timeout" ? "success" : "error";
// Make sure that the request was successful or notmodified
if ( status != "error" )
{
// process the data (runs the xml through httpData regardless of callback)
var data = jQuery.uploadHttpData( xml, s.dataType );
// If a local callback was specified, fire it and pass it the data
if ( s.success )
s.success( data, status );
// Fire the global callback
if( s.global )
jQuery.event.trigger( "ajaxSuccess", [xml, s] );
} else
jQuery.handleError(s, xml, status);
} catch(e)
{
status = "error";
jQuery.handleError(s, xml, status, e);
}
// The request was completed
if( s.global )
jQuery.event.trigger( "ajaxComplete", [xml, s] );
// Handle the global AJAX counter
if ( s.global && ! --jQuery.active )
jQuery.event.trigger( "ajaxStop" );
// Process result
if ( s.complete )
s.complete(xml, status);
jQuery(io).unbind()
setTimeout(function()
{ try
{
jQuery(io).remove();
jQuery(form).remove();
} catch(e)
{
jQuery.handleError(s, xml, null, e);
}
}, 100)
xml = null
}
}
// Timeout checker
if ( s.timeout > 0 )
{
setTimeout(function(){
// Check to see if the request is still happening
if( !requestDone ) uploadCallback( "timeout" );
}, s.timeout);
}
try
{
var form = jQuery('#' + formId);
jQuery(form).attr('action', s.url);
jQuery(form).attr('method', 'POST');
jQuery(form).attr('target', frameId);
if(form.encoding)
{
jQuery(form).attr('encoding', 'multipart/form-data');
}
else
{
jQuery(form).attr('enctype', 'multipart/form-data');
}
jQuery(form).submit();
} catch(e)
{
jQuery.handleError(s, xml, null, e);
}
jQuery('#' + frameId).load(uploadCallback);
return {abort: function(){
try
{
jQuery('#' + frameId).remove();
jQuery(form).remove();
}
catch(e){}
}};
},
uploadHttpData: function( r, type ) {
var data = !type;
data = type == "xml" || data ? r.responseXML : r.responseText;
// If the type is "script", eval it in global context
if ( type == "script" )
jQuery.globalEval( data );
// Get the JavaScript object, if JSON is used.
if ( type == "json" )
data = jQuery.parseJSON(jQuery(data).text());
// evaluate scripts within html
if ( type == "html" )
jQuery("<div>").html(data).evalScripts();
return data;
},
handleError: function( s, xml, status, e ) {
// If a local callback was specified, fire it
if ( s.error )
s.error( xml, status, e );
// Fire the global callback
if ( s.global )
jQuery.event.trigger( "ajaxError", [xml, s, e] );
}
});

View File

@ -8,6 +8,9 @@
<title>作家管理系统-小说精品屋</title>
<link rel="stylesheet" href="/css/base.css?v=1"/>
<link rel="stylesheet" href="/css/user.css"/>
<style type="text/css">
.opacity{-khtml-opacity:0.0;-moz-opacity:0.0;filter:alpha(opacity=0);filter:"alpha(opacity=0)";opacity:0.0; filter: progid:DXImageTransform.Microsoft.Alpha(opacity=0);/*兼容ie8及以下*/}
</style>
</head>
</head>
<body class="">
@ -74,8 +77,15 @@
<input type="hidden" id="catName" name="catName" value="玄幻奇幻"/>
<b>小说名:</b>
<li><input type="text" id="bookName" name="bookName" class="s_input" ></li>
<b>小说封面图片路径</b>
<li><input type="text" id="picUrl" name="picUrl" class="s_input" ></li>
<b>小说封面:</b>
<li style="position: relative">
<input class="opacity" onchange="picChange()"
type="file" id="file0" name="file"
title="点击更换图片"
style="z-index: 100;cursor: pointer;left: 0px; top: 0px; width: 100px; height: 130px; opacity: 0; position: absolute; "
/>
<img style="width:100px;height: 130px" id="picImage" src="/images/default.gif" alt="">
<input type="hidden" id="picUrl" name="picUrl" class="s_input" value="/images/default.gif"></li>
<b>小说介绍:</b>
<li><textarea name="bookDesc" rows="5" cols="53" id="bookDesc"
@ -126,11 +136,42 @@
</body>
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
<script src="/javascript/ajaxfileupload.js" type="text/javascript"></script>
<script src="/layui/layui.all.js" type="text/javascript"></script>
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
function picChange() {
var file = $("#file0").val(); //文件名称
if (file != "") {
$.ajaxFileUpload({
url : "/file/upload", //用于文件上传的服务器端请求地址
secureuri : false, //是否需要安全协议一般设置为false
fileElementId : "file0", //文件上传域的ID
dataType : "json", //返回值类型 一般设置为json
type : "post",
success : function(data) { //服务器成功响应处理函数
if (data.code == 200) {
$("#picImage").attr("src", data.data);
$("#picUrl").val(data.data);
}else {
layer.alert('图片上传失败');
}
}
});
} else {
alert("请选择上传文件");
}
}
var lock = false;
function addBook() {
if(lock){

View File

@ -159,7 +159,7 @@
if(sortBy != false){
searchData.sort = sortBy;
}else{
}else if(!keyword){
sortBy = "last_index_update_time";
}

View File

@ -58,13 +58,13 @@
<ul class="layui-nav app" lay-filter="" style="display:none;padding:0 10px;text-align: center">
<li class="layui-nav-item"><a>分类</a>
<dl class="layui-nav-child">
<dd><a href="/book/book_ranking.html?catId=1">玄幻小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=2">修真小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=3">都市小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=4">历史小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=6">网游小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=5">科幻小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=7">女频小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=1&sort=last_index_update_time">玄幻小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=2&sort=last_index_update_time">修真小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=3&sort=last_index_update_time">都市小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=4&sort=last_index_update_time">历史小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=6&sort=last_index_update_time">网游小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=5&sort=last_index_update_time">科幻小说</a></dd>
<dd><a href="/book/book_ranking.html?catId=7&sort=last_index_update_time">女频小说</a></dd>
</dl>
</li>
<li class="layui-nav-item"><a>全本</a>
@ -95,13 +95,13 @@
</ul>
<ul class="layui-nav pc" lay-filter="" style="padding:0 36px;text-align: center">
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=1">玄幻小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=2">修真小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=3">都市小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=4">历史小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=6">网游小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=5">科幻小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=7">女频小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=1&sort=last_index_update_time">玄幻小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=2&sort=last_index_update_time">修真小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=3&sort=last_index_update_time">都市小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=4&sort=last_index_update_time">历史小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=6&sort=last_index_update_time">网游小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=5&sort=last_index_update_time">科幻小说</a></li>
<li class="layui-nav-item"><a href="/book/book_ranking.html?catId=7&sort=last_index_update_time">女频小说</a></li>
<li class="layui-nav-item"><a>完本小说</a>
<dl class="layui-nav-child"> <!-- 二级菜单 -->
<dd><a href="/book/book_ranking.html?bookStatus=1">全部小说</a></dd>

View File

@ -5,7 +5,7 @@
<groupId>com.java2nb</groupId>
<artifactId>novel</artifactId>
<version>2.5.0</version>
<version>2.6.2</version>
<modules>
<module>novel-common</module>
<module>novel-front</module>

2
sql/20200608.sql Normal file
View File

@ -0,0 +1,2 @@
INSERT INTO `crawl_source` (`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES
(6, '新笔趣阁', '{\n \"bookListUrl\": \"http://www.xbiquge.la/fenlei/{catId}_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"6\",\n \"catId6\": \"5\"\n },\n \"bookIdPatten\": \"<a\\\\s+href=\\\"http://www.xbiquge.la/(\\\\d+/\\\\d+)/\\\"\\\\s+target=\\\"_blank\\\">\",\n \"pagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">(\\\\d+)/\\\\d+</em>\",\n \"totalPagePatten\": \"<em\\\\s+id=\\\"pagestats\\\">\\\\d+/(\\\\d+)</em>\",\n \"bookDetailUrl\": \"http://www.xbiquge.la/{bookId}/\",\n \"bookNamePatten\": \"<h1>([^/]+)</h1>\",\n \"authorNamePatten\": \"者:([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.xbiquge.la/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"bookStatusRule\": {},\n \"descStart\": \"<div id=\\\"intro\\\">\",\n \"descEnd\": \"</div>\",\n \"upadateTimePatten\": \"<p>最后更新:(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)</p>\",\n \"upadateTimeFormatPatten\": \"yyyy-MM-dd HH:mm:ss\",\n \"bookIndexUrl\": \"http://www.xbiquge.la/{bookId}/\",\n \"indexIdPatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/(\\\\d+)\\\\.html\'\\\\s+>[^/]+</a>\",\n \"indexNamePatten\": \"<a\\\\s+href=\'/\\\\d+/\\\\d+/\\\\d+\\\\.html\'\\\\s+>([^/]+)</a>\",\n \"bookContentUrl\": \"http://www.xbiquge.la/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\">\",\n \"contentEnd\": \"<p>\"\n}', 0, '2020-05-23 22:46:58', '2020-05-23 22:46:58');

40
sql/20200615.sql Normal file
View File

@ -0,0 +1,40 @@
/*
Navicat MySQL Data Transfer
Source Server : localhost
Source Server Version : 50725
Source Host : localhost:3306
Source Database : novel_plus
Target Server Type : MYSQL
Target Server Version : 50725
File Encoding : 65001
Date: 2020-06-15 15:06:55
*/
SET FOREIGN_KEY_CHECKS=0;
-- ----------------------------
-- Table structure for crawl_single_task
-- ----------------------------
DROP TABLE IF EXISTS `crawl_single_task`;
CREATE TABLE `crawl_single_task` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`source_id` int(11) DEFAULT NULL COMMENT '爬虫源ID',
`source_name` varchar(50) DEFAULT NULL COMMENT '爬虫源名',
`source_book_id` varchar(255) DEFAULT NULL COMMENT '源站小说ID',
`cat_id` int(11) DEFAULT NULL COMMENT '分类ID',
`book_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说名',
`author_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说作者名',
`task_status` tinyint(1) DEFAULT '2' COMMENT '任务状态0失败1成功2未执行',
`exc_count` tinyint(2) DEFAULT '0' COMMENT '已经执行次数最多执行5次',
`create_time` datetime DEFAULT NULL COMMENT '创建时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='抓取单本小说任务表';
-- ----------------------------
-- Records of crawl_single_task
-- ----------------------------
INSERT INTO `crawl_single_task` VALUES ('6', '2', '百书斋', '1', '1', '1', '1', '0', '5', '2020-06-15 14:36:07');
INSERT INTO `crawl_single_task` VALUES ('7', '5', '笔趣阁', '108_108291', '1', '衍天志之不朽仙', '白衣少年丶', '1', '1', '2020-06-15 14:46:08');

View File

@ -1809,4 +1809,30 @@ CREATE TABLE `book_content9` (
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n \"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"7\",\n \"catId6\": \"6\",\n \"catId7\": \"8\"\n },\n \"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n \"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n \"totalPagePatten\": \"\\\\d+/(\\\\d+)页\",\n \"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n \"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n \"authorNamePatten\": \"<p>作者:([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"statusPatten\": \"<p>状态:([^/]+)</p>\",\n \"bookStatusRule\": {\n \"连载中\": 0,\n \"完本\": 1\n },\n \"descStart\": \"<div class=\\\"intro_info\\\">\",\n \"descEnd\": \"最新章节推荐地址\",\n \"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n \"bookIndexStart\": \"<dt>《\",\n \"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n \"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n \"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n \"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;
DROP TABLE IF EXISTS `crawl_single_task`;
CREATE TABLE `crawl_single_task` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`source_id` int(11) DEFAULT NULL COMMENT '爬虫源ID',
`source_name` varchar(50) DEFAULT NULL COMMENT '爬虫源名',
`source_book_id` varchar(255) DEFAULT NULL COMMENT '源站小说ID',
`cat_id` int(11) DEFAULT NULL COMMENT '分类ID',
`book_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说名',
`author_name` varchar(50) DEFAULT NULL COMMENT '爬取的小说作者名',
`task_status` tinyint(1) DEFAULT '2' COMMENT '任务状态0失败1成功2未执行',
`exc_count` tinyint(2) DEFAULT '0' COMMENT '已经执行次数最多执行5次',
`create_time` datetime DEFAULT NULL COMMENT '创建时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COMMENT='抓取单本小说任务表';
-- ----------------------------
-- Records of crawl_single_task
-- ----------------------------
INSERT INTO `crawl_single_task` VALUES ('6', '2', '百书斋', '1', '1', '1', '1', '0', '5', '2020-06-15 14:36:07');
INSERT INTO `crawl_single_task` VALUES ('7', '5', '笔趣阁', '108_108291', '1', '衍天志之不朽仙', '白衣少年丶', '1', '1', '2020-06-15 14:46:08');
UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n \"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"7\",\n \"catId6\": \"6\",\n \"catId7\": \"8\"\n },\n \"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n \"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n \"totalPagePatten\": \"\\\\d+/(\\\\d+)页\",\n \"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n \"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n \"authorNamePatten\": \"<p>作者:([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"statusPatten\": \"<p>状态:([^/]+)</p>\",\n \"bookStatusRule\": {\n \"连载中\": 0,\n \"完本\": 1\n },\n \"descStart\": \"<div class=\\\"intro_info\\\">\",\n \"descEnd\": \"最新章节推荐地址\",\n \"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n \"bookIndexStart\": \"<dt>《\",\n \"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n \"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n \"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n \"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;
INSERT INTO `friend_link` ( `link_name`, `link_url`, `sort`, `is_open`, `create_user_id`, `create_time`, `update_user_id`, `update_time`) VALUES
('小羊影视', 'http://video.java2nb.com/', 11, 1, NULL, NULL, NULL, NULL),
('官方论坛', 'http://bbs.java2nb.com', 21, 1, NULL, NULL, NULL, NULL);