mirror of
https://github.com/201206030/novel-plus.git
synced 2025-07-01 15:26:37 +00:00
Compare commits
24 Commits
Author | SHA1 | Date | |
---|---|---|---|
efb136e3be | |||
7955db0e3c | |||
60dc28c5ed | |||
1534220f0c | |||
0830f6ffeb | |||
adc83db64e | |||
9c11f22816 | |||
24abe7714f | |||
a9fc80eba1 | |||
32541a7cb6 | |||
42bcecc304 | |||
a07643bde0 | |||
1f53b56bd6 | |||
2c86cb9a7d | |||
a4d6272a4f | |||
55d5deea74 | |||
4f474b91a8 | |||
ca22eed665 | |||
df1b72fb58 | |||
415bf8a64c | |||
3f009dc1f9 | |||
0e156c04b4 | |||
d4fa0abc4e | |||
eff4fc4c7c |
@ -1,5 +1,4 @@
|
|||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://www.swiftproxy.net/?code=T2WV1VT50"><img src="https://xxyopen.com/images/ad1.png" alt="AD" ></a>
|
|
||||||
<a href="https://cloud.tencent.com/act/cps/redirect?redirect=2446&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console"><img src="https://youdoc.github.io/img/tencent.jpg" alt="AD" ></a>
|
<a href="https://cloud.tencent.com/act/cps/redirect?redirect=2446&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console"><img src="https://youdoc.github.io/img/tencent.jpg" alt="AD" ></a>
|
||||||
</p>
|
</p>
|
||||||
<p align="center">
|
<p align="center">
|
||||||
@ -10,7 +9,7 @@
|
|||||||
</p>
|
</p>
|
||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
👉 <a href='https://novel.xxyopen.com'>官网</a> | 👉 <a href='https://www.bilibili.com/video/BV1Zo4y187Mi'>项目演示</a> | 👉 <a href='https://docs.xxyopen.com/course/novelplus/1.html'>安装教程</a>
|
👉 <a href='https://novel.xxyopen.com'>官网</a> | 👉 <a href='http://117.72.165.13:8888'>演示站点</a> | 👉 <a href='https://docs.xxyopen.com/course/novelplus/1.html'>安装教程</a>
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
## 项目介绍
|
## 项目介绍
|
||||||
@ -22,8 +21,8 @@ TXT 文本存储)、阅读主题切换、多爬虫源自动采集和更新数
|
|||||||
## 项目地址
|
## 项目地址
|
||||||
|
|
||||||
- 学习版:[GitHub](https://github.com/201206030/novel) | [码云](https://gitee.com/novel_dev_team/novel)
|
- 学习版:[GitHub](https://github.com/201206030/novel) | [码云](https://gitee.com/novel_dev_team/novel)
|
||||||
| [保姆级教程](https://docs.xxyopen.com)
|
| [保姆级教程](https://docs.xxyopen.com)
|
||||||
- **应用版**:[GitHub](https://github.com/201206030/novel-plus) | [码云](https://gitee.com/novel_dev_team/novel-plus)
|
- **应用版**:[GitHub](https://github.com/201206030/novel-plus) | [码云](https://gitee.com/novel_dev_team/novel-plus) | [演示站点](http://117.72.165.13:8888)
|
||||||
- 微服务版:[GitHub](https://github.com/201206030/novel-cloud) | [码云](https://gitee.com/novel_dev_team/novel-cloud)
|
- 微服务版:[GitHub](https://github.com/201206030/novel-cloud) | [码云](https://gitee.com/novel_dev_team/novel-cloud)
|
||||||
|
|
||||||
## 项目结构
|
## 项目结构
|
||||||
|
@ -11,13 +11,13 @@ dataSources:
|
|||||||
ds_1:
|
ds_1:
|
||||||
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
|
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
|
||||||
driverClassName: com.mysql.cj.jdbc.Driver
|
driverClassName: com.mysql.cj.jdbc.Driver
|
||||||
jdbcUrl: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
|
jdbcUrl: jdbc:mysql://localhost:3306/novel_plus?allowPublicKeyRetrieval=true&useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
|
||||||
username: root
|
username: root
|
||||||
password: test123456
|
password: test123456
|
||||||
ds_2:
|
ds_2:
|
||||||
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
|
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
|
||||||
driverClassName: com.mysql.cj.jdbc.Driver
|
driverClassName: com.mysql.cj.jdbc.Driver
|
||||||
url: jdbc:mysql://localhost:3306/information_schema?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
|
url: jdbc:mysql://localhost:3306/information_schema?allowPublicKeyRetrieval=true&useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
|
||||||
username: root
|
username: root
|
||||||
password: test123456
|
password: test123456
|
||||||
# 规则配置
|
# 规则配置
|
||||||
|
@ -5,7 +5,7 @@
|
|||||||
|
|
||||||
<groupId>com.java2nb</groupId>
|
<groupId>com.java2nb</groupId>
|
||||||
<artifactId>novel-admin</artifactId>
|
<artifactId>novel-admin</artifactId>
|
||||||
<version>5.1.0</version>
|
<version>5.1.5</version>
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
<name>novel-admin</name>
|
<name>novel-admin</name>
|
||||||
|
@ -5,7 +5,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<artifactId>novel</artifactId>
|
<artifactId>novel</artifactId>
|
||||||
<groupId>com.java2nb</groupId>
|
<groupId>com.java2nb</groupId>
|
||||||
<version>5.1.0</version>
|
<version>5.1.5</version>
|
||||||
</parent>
|
</parent>
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
@ -0,0 +1,53 @@
|
|||||||
|
package com.java2nb.novel.core.advice;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.module.SimpleModule;
|
||||||
|
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
|
||||||
|
import org.springframework.core.MethodParameter;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.http.converter.HttpMessageConverter;
|
||||||
|
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
|
||||||
|
import org.springframework.http.server.ServerHttpRequest;
|
||||||
|
import org.springframework.http.server.ServerHttpResponse;
|
||||||
|
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||||
|
import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice;
|
||||||
|
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在对 RestController 返回对象 json 序列化时,将所有 Long 类型转为 String 类型返回,避免前端数据精度丢失的问题
|
||||||
|
* 取代 spring.jackson.generator.write-numbers-as-strings=true 配置,避免影响全局的 ObjectMapper
|
||||||
|
*
|
||||||
|
* @author xiongxiaoyang
|
||||||
|
* */
|
||||||
|
@RestControllerAdvice
|
||||||
|
public class CustomResponseBodyAdvice implements ResponseBodyAdvice<Object> {
|
||||||
|
|
||||||
|
private final ObjectMapper customObjectMapper;
|
||||||
|
|
||||||
|
public CustomResponseBodyAdvice(Jackson2ObjectMapperBuilder builder) {
|
||||||
|
customObjectMapper = builder.createXmlMapper(false).build();
|
||||||
|
SimpleModule simpleModule = new SimpleModule();
|
||||||
|
simpleModule.addSerializer(Long.class, ToStringSerializer.instance);
|
||||||
|
simpleModule.addSerializer(Long.TYPE, ToStringSerializer.instance);
|
||||||
|
customObjectMapper.registerModule(simpleModule);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean supports(MethodParameter returnType, Class<? extends HttpMessageConverter<?>> converterType) {
|
||||||
|
// 返回 true 表示对所有 Controller 的响应都生效
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Object beforeBodyWrite(Object body, MethodParameter returnType, MediaType selectedContentType, Class<? extends HttpMessageConverter<?>> selectedConverterType, ServerHttpRequest request, ServerHttpResponse response) {
|
||||||
|
// 使用自定义的 ObjectMapper 序列化响应体
|
||||||
|
if(Objects.nonNull(body)) {
|
||||||
|
return customObjectMapper.valueToTree(body);
|
||||||
|
}else{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -41,11 +41,6 @@ public interface CacheKey {
|
|||||||
* */
|
* */
|
||||||
String TEMPLATE_DIR_KEY = "templateDirKey";;
|
String TEMPLATE_DIR_KEY = "templateDirKey";;
|
||||||
|
|
||||||
/**
|
|
||||||
* 正在运行的爬虫线程存储KEY前缀
|
|
||||||
* */
|
|
||||||
String RUNNING_CRAWL_THREAD_KEY_PREFIX = "runningCrawlTreadDataKeyPrefix";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 上一次搜索引擎更新的时间
|
* 上一次搜索引擎更新的时间
|
||||||
* */
|
* */
|
||||||
@ -69,4 +64,8 @@ public interface CacheKey {
|
|||||||
* 测试爬虫规则缓存
|
* 测试爬虫规则缓存
|
||||||
*/
|
*/
|
||||||
String BOOK_TEST_PARSE = "testParse";
|
String BOOK_TEST_PARSE = "testParse";
|
||||||
|
/**
|
||||||
|
* AI生成图片
|
||||||
|
* */
|
||||||
|
String AI_GEN_PIC = "aiGenPic";
|
||||||
}
|
}
|
||||||
|
@ -5,8 +5,8 @@ import org.springframework.http.*;
|
|||||||
import org.springframework.web.client.RestTemplate;
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
import java.nio.charset.Charset;
|
import java.nio.charset.Charset;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author Administrator
|
* @author Administrator
|
||||||
@ -16,7 +16,7 @@ public class HttpUtil {
|
|||||||
|
|
||||||
private static final String DEFAULT_CHARSET = "utf-8";
|
private static final String DEFAULT_CHARSET = "utf-8";
|
||||||
|
|
||||||
private static final Map<String, RestTemplate> REST_TEMPLATE_MAP = new HashMap<>();
|
private static final Map<String, RestTemplate> REST_TEMPLATE_MAP = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public static String getByHttpClientWithChrome(String url, String charset) {
|
public static String getByHttpClientWithChrome(String url, String charset) {
|
||||||
log.debug("Get url:{}", url);
|
log.debug("Get url:{}", url);
|
||||||
|
@ -6,11 +6,6 @@ spring:
|
|||||||
mode: LEGACYHTML5 #去除thymeleaf的html严格校验thymeleaf.mode=LEGACYHTML5
|
mode: LEGACYHTML5 #去除thymeleaf的html严格校验thymeleaf.mode=LEGACYHTML5
|
||||||
cache: false # 是否开启模板缓存,默认true,建议在开发时关闭缓存,不然没法看到实时
|
cache: false # 是否开启模板缓存,默认true,建议在开发时关闭缓存,不然没法看到实时
|
||||||
|
|
||||||
# 将所有数字转为 String 类型返回,避免前端数据精度丢失的问题
|
|
||||||
jackson:
|
|
||||||
generator:
|
|
||||||
write-numbers-as-strings: true
|
|
||||||
|
|
||||||
#上传文件的最大值(100M)
|
#上传文件的最大值(100M)
|
||||||
servlet:
|
servlet:
|
||||||
multipart:
|
multipart:
|
||||||
|
@ -5,7 +5,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<artifactId>novel</artifactId>
|
<artifactId>novel</artifactId>
|
||||||
<groupId>com.java2nb</groupId>
|
<groupId>com.java2nb</groupId>
|
||||||
<version>5.1.0</version>
|
<version>5.1.5</version>
|
||||||
</parent>
|
</parent>
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
@ -7,6 +7,6 @@ import com.java2nb.novel.entity.Book;
|
|||||||
* */
|
* */
|
||||||
public interface CrawlBookHandler {
|
public interface CrawlBookHandler {
|
||||||
|
|
||||||
void handle(Book book);
|
void handle(Book book) throws InterruptedException;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -10,9 +10,11 @@ import com.java2nb.novel.utils.CrawlHttpClient;
|
|||||||
import io.github.xxyopen.util.IdWorker;
|
import io.github.xxyopen.util.IdWorker;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.text.ParseException;
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
@ -26,6 +28,7 @@ import java.util.regex.Pattern;
|
|||||||
*
|
*
|
||||||
* @author Administrator
|
* @author Administrator
|
||||||
*/
|
*/
|
||||||
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class CrawlParser {
|
public class CrawlParser {
|
||||||
@ -34,8 +37,8 @@ public class CrawlParser {
|
|||||||
|
|
||||||
private final CrawlHttpClient crawlHttpClient;
|
private final CrawlHttpClient crawlHttpClient;
|
||||||
|
|
||||||
@SneakyThrows
|
public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler)
|
||||||
public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler) {
|
throws InterruptedException {
|
||||||
Book book = new Book();
|
Book book = new Book();
|
||||||
String bookDetailUrl = ruleBean.getBookDetailUrl().replace("{bookId}", bookId);
|
String bookDetailUrl = ruleBean.getBookDetailUrl().replace("{bookId}", bookId);
|
||||||
String bookDetailHtml = crawlHttpClient.get(bookDetailUrl, ruleBean.getCharset());
|
String bookDetailHtml = crawlHttpClient.get(bookDetailUrl, ruleBean.getCharset());
|
||||||
@ -97,6 +100,22 @@ public class CrawlParser {
|
|||||||
.replaceAll("<p>\\s*</p>", "")
|
.replaceAll("<p>\\s*</p>", "")
|
||||||
.replaceAll("<p>", "")
|
.replaceAll("<p>", "")
|
||||||
.replaceAll("</p>", "<br/>");
|
.replaceAll("</p>", "<br/>");
|
||||||
|
// 小说简介过滤
|
||||||
|
String filterDesc = ruleBean.getFilterDesc();
|
||||||
|
if (StringUtils.isNotBlank(filterDesc)) {
|
||||||
|
String[] filterRules = filterDesc.replace("\r\n", "\n").split("\n");
|
||||||
|
for (String filterRule : filterRules) {
|
||||||
|
if (StringUtils.isNotBlank(filterRule)) {
|
||||||
|
desc = desc.replaceAll(filterRule, "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 去除小说简介前后空格
|
||||||
|
desc = desc.trim();
|
||||||
|
// 去除小说简介末尾冗余的小说名
|
||||||
|
if (desc.endsWith(bookName)) {
|
||||||
|
desc = desc.substring(0, desc.length() - bookName.length());
|
||||||
|
}
|
||||||
//设置书籍简介
|
//设置书籍简介
|
||||||
book.setBookDesc(desc);
|
book.setBookDesc(desc);
|
||||||
if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {
|
if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {
|
||||||
@ -120,8 +139,12 @@ public class CrawlParser {
|
|||||||
if (isFindUpdateTime) {
|
if (isFindUpdateTime) {
|
||||||
String updateTime = updateTimeMatch.group(1);
|
String updateTime = updateTimeMatch.group(1);
|
||||||
//设置更新时间
|
//设置更新时间
|
||||||
book.setLastIndexUpdateTime(
|
try {
|
||||||
new SimpleDateFormat(ruleBean.getUpadateTimeFormatPatten()).parse(updateTime));
|
book.setLastIndexUpdateTime(
|
||||||
|
new SimpleDateFormat(ruleBean.getUpadateTimeFormatPatten()).parse(updateTime));
|
||||||
|
} catch (ParseException e) {
|
||||||
|
log.error("解析最新章节更新时间出错", e);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -144,7 +167,7 @@ public class CrawlParser {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean,
|
public boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean,
|
||||||
Map<Integer, BookIndex> existBookIndexMap, CrawlBookChapterHandler handler) {
|
Map<Integer, BookIndex> existBookIndexMap, CrawlBookChapterHandler handler) throws InterruptedException {
|
||||||
|
|
||||||
Date currentDate = new Date();
|
Date currentDate = new Date();
|
||||||
|
|
||||||
@ -231,6 +254,8 @@ public class CrawlParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// 去除小说内容末尾的所有换行
|
||||||
|
content = removeTrailingBrTags(content);
|
||||||
//插入章节目录和章节内容
|
//插入章节目录和章节内容
|
||||||
BookIndex bookIndex = new BookIndex();
|
BookIndex bookIndex = new BookIndex();
|
||||||
bookIndex.setIndexName(indexName);
|
bookIndex.setIndexName(indexName);
|
||||||
@ -307,4 +332,12 @@ public class CrawlParser {
|
|||||||
return false;
|
return false;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除字符串末尾的所有 <br> 类似标签(允许各种空格)
|
||||||
|
*/
|
||||||
|
public static String removeTrailingBrTags(String str) {
|
||||||
|
return str.replaceAll("(?i)(?:\\s*<\\s*br\\s*/?\\s*>)++(?:\\s|\\u3000)*$", "");
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -45,6 +45,7 @@ public class RuleBean {
|
|||||||
private String visitCountPatten;
|
private String visitCountPatten;
|
||||||
private String descStart;
|
private String descStart;
|
||||||
private String descEnd;
|
private String descEnd;
|
||||||
|
private String filterDesc;
|
||||||
private String upadateTimePatten;
|
private String upadateTimePatten;
|
||||||
private String upadateTimeFormatPatten;
|
private String upadateTimeFormatPatten;
|
||||||
private String bookIndexUrl;
|
private String bookIndexUrl;
|
||||||
|
@ -74,10 +74,8 @@ public class StarterListener implements ServletContextInitializer {
|
|||||||
needUpdateBook.getId());
|
needUpdateBook.getId());
|
||||||
//解析章节目录
|
//解析章节目录
|
||||||
crawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(), book,
|
crawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(), book,
|
||||||
ruleBean, existBookIndexMap, chapter -> {
|
ruleBean, existBookIndexMap, chapter -> bookService.updateBookAndIndexAndContent(book, chapter.getBookIndexList(),
|
||||||
bookService.updateBookAndIndexAndContent(book, chapter.getBookIndexList(),
|
chapter.getBookContentList(), existBookIndexMap));
|
||||||
chapter.getBookContentList(), existBookIndexMap);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error(e.getMessage(), e);
|
log.error(e.getMessage(), e);
|
||||||
|
@ -1,61 +0,0 @@
|
|||||||
package com.java2nb.novel.core.schedule;
|
|
||||||
|
|
||||||
|
|
||||||
import com.java2nb.novel.core.cache.CacheKey;
|
|
||||||
import com.java2nb.novel.core.cache.CacheService;
|
|
||||||
import com.java2nb.novel.entity.CrawlSource;
|
|
||||||
import com.java2nb.novel.service.CrawlService;
|
|
||||||
import io.github.xxyopen.util.ThreadUtil;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 爬虫线程监控器,监控执行完成的爬虫源,并修改状态
|
|
||||||
*
|
|
||||||
* @author Administrator
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
@Slf4j
|
|
||||||
public class CrawlThreadMonitor {
|
|
||||||
|
|
||||||
private final CacheService cacheService;
|
|
||||||
|
|
||||||
private final CrawlService crawlService;
|
|
||||||
|
|
||||||
@Scheduled(fixedRate = 1000 * 60 * 5)
|
|
||||||
public void monitor() {
|
|
||||||
|
|
||||||
//查询需要监控的正在运行的爬虫源
|
|
||||||
List<CrawlSource> sources = crawlService.queryCrawlSourceByStatus((byte) 1);
|
|
||||||
|
|
||||||
for (CrawlSource source : sources) {
|
|
||||||
Set<Long> runningCrawlThreadIds = (Set<Long>) cacheService.getObject(CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + source.getId());
|
|
||||||
boolean sourceStop = true;
|
|
||||||
if (runningCrawlThreadIds != null) {
|
|
||||||
for (Long threadId : runningCrawlThreadIds) {
|
|
||||||
Thread thread = ThreadUtil.findThread(threadId);
|
|
||||||
|
|
||||||
if (thread != null && thread.isAlive()) {
|
|
||||||
//有活跃线程,说明该爬虫源正在运行,数据库中状态正确,不需要修改
|
|
||||||
sourceStop = false;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (sourceStop) {
|
|
||||||
crawlService.updateCrawlSourceStatus(source.getId(), (byte) 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
@ -53,7 +53,7 @@ public interface CrawlService {
|
|||||||
* @param ruleBean 采集规则\
|
* @param ruleBean 采集规则\
|
||||||
* @return true:成功,false:失败
|
* @return true:成功,false:失败
|
||||||
* */
|
* */
|
||||||
boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId);
|
boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) throws InterruptedException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 根据爬虫状态查询爬虫源集合
|
* 根据爬虫状态查询爬虫源集合
|
||||||
|
@ -34,6 +34,7 @@ import org.mybatis.dynamic.sql.render.RenderingStrategies;
|
|||||||
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
|
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
@ -65,6 +66,10 @@ public class CrawlServiceImpl implements CrawlService {
|
|||||||
|
|
||||||
private final CrawlHttpClient crawlHttpClient;
|
private final CrawlHttpClient crawlHttpClient;
|
||||||
|
|
||||||
|
private final Map<Integer, Byte> crawlSourceStatusMap = new HashMap<>();
|
||||||
|
|
||||||
|
private final Map<Integer, Set<Long>> runningCrawlThread = new HashMap<>();
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void addCrawlSource(CrawlSource source) {
|
public void addCrawlSource(CrawlSource source) {
|
||||||
@ -103,6 +108,8 @@ public class CrawlServiceImpl implements CrawlService {
|
|||||||
.build()
|
.build()
|
||||||
.render(RenderingStrategies.MYBATIS3);
|
.render(RenderingStrategies.MYBATIS3);
|
||||||
List<CrawlSource> crawlSources = crawlSourceMapper.selectMany(render);
|
List<CrawlSource> crawlSources = crawlSourceMapper.selectMany(render);
|
||||||
|
crawlSources.forEach(crawlSource -> crawlSource.setSourceStatus(
|
||||||
|
Optional.ofNullable(crawlSourceStatusMap.get(crawlSource.getId())).orElse((byte) 0)));
|
||||||
PageBean<CrawlSource> pageBean = PageBuilder.build(crawlSources);
|
PageBean<CrawlSource> pageBean = PageBuilder.build(crawlSources);
|
||||||
pageBean.setList(BeanUtil.copyList(crawlSources, CrawlSourceVO.class));
|
pageBean.setList(BeanUtil.copyList(crawlSources, CrawlSourceVO.class));
|
||||||
return pageBean;
|
return pageBean;
|
||||||
@ -112,14 +119,13 @@ public class CrawlServiceImpl implements CrawlService {
|
|||||||
@Override
|
@Override
|
||||||
public void openOrCloseCrawl(Integer sourceId, Byte sourceStatus) {
|
public void openOrCloseCrawl(Integer sourceId, Byte sourceStatus) {
|
||||||
|
|
||||||
//判断是开启还是关闭,如果是关闭,则修改数据库状态后获取该爬虫正在运行的线程集合并全部停止
|
// 判断是开启还是关闭,如果是关闭,则获取该爬虫源正在运行的线程集合并全部中断
|
||||||
//如果是开启,先查询数据库中状态,判断该爬虫源是否还在运行,如果在运行,则忽略,
|
// 如果是开启,先判断该爬虫源是否还在运行,如果在运行,则忽略,如果没有运行则启动线程爬取小说数据并加入到runningCrawlThread中
|
||||||
// 如果没有则修改数据库状态,并启动线程爬取小说数据加入到runningCrawlThread中
|
// 最后,保存爬虫源状态
|
||||||
if (sourceStatus == (byte) 0) {
|
if (sourceStatus == (byte) 0) {
|
||||||
//关闭,直接修改数据库状态,并直接修改数据库状态后获取该爬虫正在运行的线程集合全部停止
|
// 关闭
|
||||||
SpringUtil.getBean(CrawlService.class).updateCrawlSourceStatus(sourceId, sourceStatus);
|
// 将该爬虫源正在运行的线程集合全部停止
|
||||||
Set<Long> runningCrawlThreadId = (Set<Long>) cacheService.getObject(
|
Set<Long> runningCrawlThreadId = runningCrawlThread.get(sourceId);
|
||||||
CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + sourceId);
|
|
||||||
if (runningCrawlThreadId != null) {
|
if (runningCrawlThreadId != null) {
|
||||||
for (Long ThreadId : runningCrawlThreadId) {
|
for (Long ThreadId : runningCrawlThreadId) {
|
||||||
Thread thread = ThreadUtil.findThread(ThreadId);
|
Thread thread = ThreadUtil.findThread(ThreadId);
|
||||||
@ -131,16 +137,13 @@ public class CrawlServiceImpl implements CrawlService {
|
|||||||
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
//开启
|
// 开启
|
||||||
//查询爬虫源状态和规则
|
Byte realSourceStatus = Optional.ofNullable(crawlSourceStatusMap.get(sourceId)).orElse((byte) 0);
|
||||||
CrawlSource source = queryCrawlSource(sourceId);
|
|
||||||
Byte realSourceStatus = source.getSourceStatus();
|
|
||||||
|
|
||||||
if (realSourceStatus == (byte) 0) {
|
if (realSourceStatus == (byte) 0) {
|
||||||
//该爬虫源已经停止运行了,修改数据库状态,并启动线程爬取小说数据加入到runningCrawlThread中
|
// 查询爬虫源规则
|
||||||
SpringUtil.getBean(CrawlService.class).updateCrawlSourceStatus(sourceId, sourceStatus);
|
CrawlSource source = queryCrawlSource(sourceId);
|
||||||
|
//该爬虫源已经停止运行了,启动线程爬取小说数据并将线程加入到runningCrawlThread中
|
||||||
RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
|
RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
|
||||||
|
|
||||||
Set<Long> threadIds = new HashSet<>();
|
Set<Long> threadIds = new HashSet<>();
|
||||||
//按分类开始爬虫解析任务
|
//按分类开始爬虫解析任务
|
||||||
for (int i = 1; i < 8; i++) {
|
for (int i = 1; i < 8; i++) {
|
||||||
@ -149,16 +152,15 @@ public class CrawlServiceImpl implements CrawlService {
|
|||||||
thread.start();
|
thread.start();
|
||||||
//thread加入到监控缓存中
|
//thread加入到监控缓存中
|
||||||
threadIds.add(thread.getId());
|
threadIds.add(thread.getId());
|
||||||
|
|
||||||
}
|
}
|
||||||
cacheService.setObject(CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + sourceId, threadIds);
|
runningCrawlThread.put(sourceId, threadIds);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 保存爬虫源状态
|
||||||
|
crawlSourceStatusMap.put(sourceId, sourceStatus);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -248,6 +250,11 @@ public class CrawlServiceImpl implements CrawlService {
|
|||||||
@Override
|
@Override
|
||||||
public void parseBookList(int catId, RuleBean ruleBean, Integer sourceId) {
|
public void parseBookList(int catId, RuleBean ruleBean, Integer sourceId) {
|
||||||
|
|
||||||
|
String catIdRule = ruleBean.getCatIdRule().get("catId" + catId);
|
||||||
|
if (StringUtils.isBlank(catIdRule)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
//当前页码1
|
//当前页码1
|
||||||
int page = 1;
|
int page = 1;
|
||||||
int totalPage = page;
|
int totalPage = page;
|
||||||
@ -255,73 +262,91 @@ public class CrawlServiceImpl implements CrawlService {
|
|||||||
while (page <= totalPage) {
|
while (page <= totalPage) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
String catIdRule = ruleBean.getCatIdRule().get("catId" + catId);
|
String catBookListUrl;
|
||||||
if (StringUtils.isNotBlank(catIdRule)) {
|
if (StringUtils.isNotBlank(ruleBean.getBookListUrl())) {
|
||||||
String catBookListUrl = "";
|
// 兼容老规则
|
||||||
if (StringUtils.isNotBlank(ruleBean.getBookListUrl())) {
|
// 拼接分类URL
|
||||||
// 兼容老规则
|
catBookListUrl = ruleBean.getBookListUrl()
|
||||||
// 拼接分类URL
|
.replace("{catId}", catIdRule)
|
||||||
catBookListUrl = ruleBean.getBookListUrl()
|
.replace("{page}", page + "");
|
||||||
.replace("{catId}", catIdRule)
|
} else {
|
||||||
.replace("{page}", page + "");
|
// 新规则
|
||||||
} else {
|
// 拼接分类URL
|
||||||
// 新规则
|
catBookListUrl = catIdRule.replace("{page}", page + "");
|
||||||
// 拼接分类URL
|
}
|
||||||
catBookListUrl = catIdRule.replace("{page}", page + "");
|
log.info("catBookListUrl:{}", catBookListUrl);
|
||||||
}
|
|
||||||
log.info("catBookListUrl:{}", catBookListUrl);
|
|
||||||
|
|
||||||
String bookListHtml = crawlHttpClient.get(catBookListUrl, ruleBean.getCharset());
|
String bookListHtml = crawlHttpClient.get(catBookListUrl, ruleBean.getCharset());
|
||||||
if (bookListHtml != null) {
|
if (bookListHtml != null) {
|
||||||
Pattern bookIdPatten = Pattern.compile(ruleBean.getBookIdPatten());
|
Pattern bookIdPatten = Pattern.compile(ruleBean.getBookIdPatten());
|
||||||
Matcher bookIdMatcher = bookIdPatten.matcher(bookListHtml);
|
Matcher bookIdMatcher = bookIdPatten.matcher(bookListHtml);
|
||||||
boolean isFindBookId = bookIdMatcher.find();
|
boolean isFindBookId = bookIdMatcher.find();
|
||||||
while (isFindBookId) {
|
while (isFindBookId) {
|
||||||
try {
|
try {
|
||||||
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||||
//捕获中断异常InterruptedException来退出线程。
|
//捕获中断异常InterruptedException来退出线程。
|
||||||
//2.非阻塞过程中通过判断中断标志来退出线程。
|
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||||
if (Thread.currentThread().isInterrupted()) {
|
if (Thread.currentThread().isInterrupted()) {
|
||||||
return;
|
return;
|
||||||
}
|
|
||||||
|
|
||||||
String bookId = bookIdMatcher.group(1);
|
|
||||||
parseBookAndSave(catId, ruleBean, sourceId, bookId);
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error(e.getMessage(), e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
isFindBookId = bookIdMatcher.find();
|
String bookId = bookIdMatcher.group(1);
|
||||||
|
parseBookAndSave(catId, ruleBean, sourceId, bookId);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
|
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||||
|
//捕获中断异常InterruptedException来退出线程。
|
||||||
|
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||||
|
return;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
|
||||||
Pattern totalPagePatten = Pattern.compile(ruleBean.getTotalPagePatten());
|
isFindBookId = bookIdMatcher.find();
|
||||||
Matcher totalPageMatcher = totalPagePatten.matcher(bookListHtml);
|
}
|
||||||
boolean isFindTotalPage = totalPageMatcher.find();
|
|
||||||
if (isFindTotalPage) {
|
|
||||||
|
|
||||||
totalPage = Integer.parseInt(totalPageMatcher.group(1));
|
Pattern totalPagePatten = Pattern.compile(ruleBean.getTotalPagePatten());
|
||||||
|
Matcher totalPageMatcher = totalPagePatten.matcher(bookListHtml);
|
||||||
}
|
boolean isFindTotalPage = totalPageMatcher.find();
|
||||||
|
if (isFindTotalPage) {
|
||||||
|
|
||||||
|
totalPage = Integer.parseInt(totalPageMatcher.group(1));
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
|
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||||
|
//捕获中断异常InterruptedException来退出线程。
|
||||||
|
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||||
|
return;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error(e.getMessage(), e);
|
log.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
if (page == totalPage) {
|
if (page >= totalPage) {
|
||||||
// 第一遍采集完成,翻到第一页,继续第二次采集,适用于分页数比较少的最近更新列表
|
// 第一遍采集完成,翻到第一页,继续第二次采集,适用于分页数比较少的最近更新列表
|
||||||
page = 0;
|
page = 1;
|
||||||
|
try {
|
||||||
|
// 第一遍采集完成,休眠1分钟
|
||||||
|
Thread.sleep(Duration.ofMinutes(1));
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
|
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||||
|
//捕获中断异常InterruptedException来退出线程。
|
||||||
|
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
page += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
page += 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) {
|
public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId)
|
||||||
|
throws InterruptedException {
|
||||||
|
|
||||||
final AtomicBoolean parseResult = new AtomicBoolean(false);
|
final AtomicBoolean parseResult = new AtomicBoolean(false);
|
||||||
|
|
||||||
@ -385,4 +410,5 @@ public class CrawlServiceImpl implements CrawlService {
|
|||||||
.render(RenderingStrategies.MYBATIS3);
|
.render(RenderingStrategies.MYBATIS3);
|
||||||
return crawlSourceMapper.selectMany(render);
|
return crawlSourceMapper.selectMany(render);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -25,13 +25,9 @@ public class CrawlHttpClient {
|
|||||||
|
|
||||||
private static final ThreadLocal<Integer> RETRY_COUNT = new ThreadLocal<>();
|
private static final ThreadLocal<Integer> RETRY_COUNT = new ThreadLocal<>();
|
||||||
|
|
||||||
public String get(String url, String charset) {
|
public String get(String url, String charset) throws InterruptedException {
|
||||||
if (Objects.nonNull(intervalMin) && Objects.nonNull(intervalMax) && intervalMax > intervalMin) {
|
if (Objects.nonNull(intervalMin) && Objects.nonNull(intervalMax) && intervalMax > intervalMin) {
|
||||||
try {
|
Thread.sleep(random.nextInt(intervalMax - intervalMin + 1) + intervalMin);
|
||||||
Thread.sleep(random.nextInt(intervalMax - intervalMin + 1) + intervalMin);
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
log.error(e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
String body = HttpUtil.getByHttpClientWithChrome(url, charset);
|
String body = HttpUtil.getByHttpClientWithChrome(url, charset);
|
||||||
if (Objects.isNull(body) || body.length() < Constants.INVALID_HTML_LENGTH) {
|
if (Objects.isNull(body) || body.length() < Constants.INVALID_HTML_LENGTH) {
|
||||||
@ -41,7 +37,7 @@ public class CrawlHttpClient {
|
|||||||
return body;
|
return body;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String processErrorHttpResult(String url, String charset) {
|
private String processErrorHttpResult(String url, String charset) throws InterruptedException{
|
||||||
Integer count = RETRY_COUNT.get();
|
Integer count = RETRY_COUNT.get();
|
||||||
if (count == null) {
|
if (count == null) {
|
||||||
count = 0;
|
count = 0;
|
||||||
|
@ -118,6 +118,9 @@
|
|||||||
示例:<b></p></b>
|
示例:<b></p></b>
|
||||||
<li><input type="text" id="descEnd" class="s_input icon_key" placeholder="小说简介结束截取字符串:">
|
<li><input type="text" id="descEnd" class="s_input icon_key" placeholder="小说简介结束截取字符串:">
|
||||||
</li>
|
</li>
|
||||||
|
示例:<b><span\s+class="allshow">([^/]+)</span></b>
|
||||||
|
<li><textarea id="filterDesc"
|
||||||
|
placeholder="过滤简介(多个内容换行)" rows="5" cols="52"></textarea></li>
|
||||||
示例:<b>更新:(\d+-\d+-\d+\s\d+:\d+:\d+)</a></b>
|
示例:<b>更新:(\d+-\d+-\d+\s\d+:\d+:\d+)</a></b>
|
||||||
<li><input type="text" id="upadateTimePatten" class="s_input icon_key"
|
<li><input type="text" id="upadateTimePatten" class="s_input icon_key"
|
||||||
placeholder="小说更新时间的正则表达式:"></li>
|
placeholder="小说更新时间的正则表达式:"></li>
|
||||||
@ -338,6 +341,9 @@
|
|||||||
|
|
||||||
crawlRule.descEnd = descEnd;
|
crawlRule.descEnd = descEnd;
|
||||||
|
|
||||||
|
var filterDesc = $("#filterDesc").val();
|
||||||
|
crawlRule.filterDesc = filterDesc;
|
||||||
|
|
||||||
var upadateTimePatten = $("#upadateTimePatten").val();
|
var upadateTimePatten = $("#upadateTimePatten").val();
|
||||||
|
|
||||||
if (upadateTimePatten.length > 0) {
|
if (upadateTimePatten.length > 0) {
|
||||||
|
@ -119,6 +119,9 @@
|
|||||||
示例:<b></p></b>
|
示例:<b></p></b>
|
||||||
<li><input type="text" id="descEnd" class="s_input icon_key" placeholder="小说简介结束截取字符串:">
|
<li><input type="text" id="descEnd" class="s_input icon_key" placeholder="小说简介结束截取字符串:">
|
||||||
</li>
|
</li>
|
||||||
|
示例:<b><span\s+class="allshow">([^/]+)</span></b>
|
||||||
|
<li><textarea id="filterDesc"
|
||||||
|
placeholder="过滤简介(多个内容换行)" rows="5" cols="52"></textarea></li>
|
||||||
示例:<b>更新:(\d+-\d+-\d+\s\d+:\d+:\d+)</a></b>
|
示例:<b>更新:(\d+-\d+-\d+\s\d+:\d+:\d+)</a></b>
|
||||||
<li><input type="text" id="upadateTimePatten" class="s_input icon_key"
|
<li><input type="text" id="upadateTimePatten" class="s_input icon_key"
|
||||||
placeholder="小说更新时间的正则表达式:"></li>
|
placeholder="小说更新时间的正则表达式:"></li>
|
||||||
@ -266,6 +269,7 @@
|
|||||||
$("#visitCountPatten").val(crawlRule.visitCountPatten);
|
$("#visitCountPatten").val(crawlRule.visitCountPatten);
|
||||||
$("#descStart").val(crawlRule.descStart);
|
$("#descStart").val(crawlRule.descStart);
|
||||||
$("#descEnd").val(crawlRule.descEnd);
|
$("#descEnd").val(crawlRule.descEnd);
|
||||||
|
$("#filterDesc").val(crawlRule.filterDesc);
|
||||||
$("#upadateTimePatten").val(crawlRule.upadateTimePatten);
|
$("#upadateTimePatten").val(crawlRule.upadateTimePatten);
|
||||||
$("#upadateTimeFormatPatten").val(crawlRule.upadateTimeFormatPatten);
|
$("#upadateTimeFormatPatten").val(crawlRule.upadateTimeFormatPatten);
|
||||||
$("#bookIndexUrl").val(crawlRule.bookIndexUrl);
|
$("#bookIndexUrl").val(crawlRule.bookIndexUrl);
|
||||||
@ -424,6 +428,9 @@
|
|||||||
|
|
||||||
crawlRule.descEnd = descEnd;
|
crawlRule.descEnd = descEnd;
|
||||||
|
|
||||||
|
var filterDesc = $("#filterDesc").val();
|
||||||
|
crawlRule.filterDesc = filterDesc;
|
||||||
|
|
||||||
var upadateTimePatten = $("#upadateTimePatten").val();
|
var upadateTimePatten = $("#upadateTimePatten").val();
|
||||||
|
|
||||||
if (upadateTimePatten.length > 0) {
|
if (upadateTimePatten.length > 0) {
|
||||||
|
@ -5,7 +5,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<artifactId>novel</artifactId>
|
<artifactId>novel</artifactId>
|
||||||
<groupId>com.java2nb</groupId>
|
<groupId>com.java2nb</groupId>
|
||||||
<version>5.1.0</version>
|
<version>5.1.5</version>
|
||||||
</parent>
|
</parent>
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
@ -17,7 +17,14 @@ import jakarta.servlet.http.HttpServletRequest;
|
|||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.ai.chat.client.ChatClient;
|
import org.springframework.ai.chat.client.ChatClient;
|
||||||
|
import org.springframework.ai.chat.messages.UserMessage;
|
||||||
|
import org.springframework.ai.chat.model.ChatModel;
|
||||||
|
import org.springframework.ai.chat.model.ChatResponse;
|
||||||
|
import org.springframework.ai.chat.prompt.Prompt;
|
||||||
|
import org.springframework.ai.openai.OpenAiChatModel;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
import org.springframework.web.bind.annotation.*;
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
import reactor.core.publisher.Flux;
|
||||||
|
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
|
|
||||||
@ -28,7 +35,7 @@ import java.util.Date;
|
|||||||
@RestController
|
@RestController
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class AuthorController extends BaseController{
|
public class AuthorController extends BaseController {
|
||||||
|
|
||||||
private final AuthorService authorService;
|
private final AuthorService authorService;
|
||||||
|
|
||||||
@ -36,62 +43,64 @@ public class AuthorController extends BaseController{
|
|||||||
|
|
||||||
private final ChatClient chatClient;
|
private final ChatClient chatClient;
|
||||||
|
|
||||||
|
private final OpenAiChatModel chatModel;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 校验笔名是否存在
|
* 校验笔名是否存在
|
||||||
* */
|
*/
|
||||||
@GetMapping("checkPenName")
|
@GetMapping("checkPenName")
|
||||||
public RestResult<Boolean> checkPenName(String penName){
|
public RestResult<Boolean> checkPenName(String penName) {
|
||||||
|
|
||||||
return RestResult.ok(authorService.checkPenName(penName));
|
return RestResult.ok(authorService.checkPenName(penName));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 作家发布小说分页列表查询
|
* 作家发布小说分页列表查询
|
||||||
* */
|
*/
|
||||||
@GetMapping("listBookByPage")
|
@GetMapping("listBookByPage")
|
||||||
public RestResult<PageBean<Book>> listBookByPage(@RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "10") int pageSize , HttpServletRequest request){
|
public RestResult<PageBean<Book>> listBookByPage(@RequestParam(value = "curr", defaultValue = "1") int page,
|
||||||
|
@RequestParam(value = "limit", defaultValue = "10") int pageSize, HttpServletRequest request) {
|
||||||
|
|
||||||
return RestResult.ok(bookService.listBookPageByUserId(getUserDetails(request).getId(),page,pageSize));
|
return RestResult.ok(bookService.listBookPageByUserId(getUserDetails(request).getId(), page, pageSize));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 发布小说
|
* 发布小说
|
||||||
* */
|
*/
|
||||||
@PostMapping("addBook")
|
@PostMapping("addBook")
|
||||||
public RestResult<Void> addBook(@RequestParam("bookDesc") String bookDesc, Book book, HttpServletRequest request){
|
public RestResult<Void> addBook(@RequestParam("bookDesc") String bookDesc, Book book, HttpServletRequest request) {
|
||||||
|
|
||||||
Author author = checkAuthor(request);
|
Author author = checkAuthor(request);
|
||||||
|
|
||||||
//bookDesc不能使用book对象来接收,否则会自动去掉前面的空格
|
//bookDesc不能使用book对象来接收,否则会自动去掉前面的空格
|
||||||
book.setBookDesc(bookDesc
|
book.setBookDesc(bookDesc
|
||||||
.replaceAll("\\n","<br>")
|
.replaceAll("\\n", "<br>")
|
||||||
.replaceAll("\\s"," "));
|
.replaceAll("\\s", " "));
|
||||||
//发布小说
|
//发布小说
|
||||||
bookService.addBook(book,author.getId(),author.getPenName());
|
bookService.addBook(book, author.getId(), author.getPenName());
|
||||||
|
|
||||||
return RestResult.ok();
|
return RestResult.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 更新小说状态,上架或下架
|
* 更新小说状态,上架或下架
|
||||||
* */
|
*/
|
||||||
@PostMapping("updateBookStatus")
|
@PostMapping("updateBookStatus")
|
||||||
public RestResult<Void> updateBookStatus(Long bookId,Byte status,HttpServletRequest request){
|
public RestResult<Void> updateBookStatus(Long bookId, Byte status, HttpServletRequest request) {
|
||||||
Author author = checkAuthor(request);
|
Author author = checkAuthor(request);
|
||||||
|
|
||||||
//更新小说状态,上架或下架
|
//更新小说状态,上架或下架
|
||||||
bookService.updateBookStatus(bookId,status,author.getId());
|
bookService.updateBookStatus(bookId, status, author.getId());
|
||||||
|
|
||||||
return RestResult.ok();
|
return RestResult.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除章节
|
* 删除章节
|
||||||
*/
|
*/
|
||||||
@DeleteMapping("deleteIndex/{indexId}")
|
@DeleteMapping("deleteIndex/{indexId}")
|
||||||
public RestResult<Void> deleteIndex(@PathVariable("indexId") Long indexId, HttpServletRequest request) {
|
public RestResult<Void> deleteIndex(@PathVariable("indexId") Long indexId, HttpServletRequest request) {
|
||||||
|
|
||||||
Author author = checkAuthor(request);
|
Author author = checkAuthor(request);
|
||||||
|
|
||||||
@ -105,7 +114,7 @@ public class AuthorController extends BaseController{
|
|||||||
* 更新章节名
|
* 更新章节名
|
||||||
*/
|
*/
|
||||||
@PostMapping("updateIndexName")
|
@PostMapping("updateIndexName")
|
||||||
public RestResult<Void> updateIndexName(Long indexId, String indexName, HttpServletRequest request) {
|
public RestResult<Void> updateIndexName(Long indexId, String indexName, HttpServletRequest request) {
|
||||||
|
|
||||||
Author author = checkAuthor(request);
|
Author author = checkAuthor(request);
|
||||||
|
|
||||||
@ -116,19 +125,18 @@ public class AuthorController extends BaseController{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 发布章节内容
|
* 发布章节内容
|
||||||
*/
|
*/
|
||||||
@PostMapping("addBookContent")
|
@PostMapping("addBookContent")
|
||||||
public RestResult<Void> addBookContent(Long bookId, String indexName, String content,Byte isVip, HttpServletRequest request) {
|
public RestResult<Void> addBookContent(Long bookId, String indexName, String content, Byte isVip,
|
||||||
|
HttpServletRequest request) {
|
||||||
Author author = checkAuthor(request);
|
Author author = checkAuthor(request);
|
||||||
|
|
||||||
content = content.replaceAll("\\n", "<br>")
|
content = content.replaceAll("\\n", "<br>")
|
||||||
.replaceAll("\\s", " ");
|
.replaceAll("\\s", " ");
|
||||||
//发布章节内容
|
//发布章节内容
|
||||||
bookService.addBookContent(bookId, indexName, content,isVip, author.getId());
|
bookService.addBookContent(bookId, indexName, content, isVip, author.getId());
|
||||||
|
|
||||||
return RestResult.ok();
|
return RestResult.ok();
|
||||||
}
|
}
|
||||||
@ -137,14 +145,14 @@ public class AuthorController extends BaseController{
|
|||||||
* 查询章节内容
|
* 查询章节内容
|
||||||
*/
|
*/
|
||||||
@GetMapping("queryIndexContent/{indexId}")
|
@GetMapping("queryIndexContent/{indexId}")
|
||||||
public RestResult<String> queryIndexContent(@PathVariable("indexId") Long indexId, HttpServletRequest request) {
|
public RestResult<String> queryIndexContent(@PathVariable("indexId") Long indexId, HttpServletRequest request) {
|
||||||
|
|
||||||
Author author = checkAuthor(request);
|
Author author = checkAuthor(request);
|
||||||
|
|
||||||
String content = bookService.queryIndexContent(indexId, author.getId());
|
String content = bookService.queryIndexContent(indexId, author.getId());
|
||||||
|
|
||||||
content = content.replaceAll("<br>", "\n")
|
content = content.replaceAll("<br>", "\n")
|
||||||
.replaceAll(" ", " ");
|
.replaceAll(" ", " ");
|
||||||
|
|
||||||
return RestResult.ok(content);
|
return RestResult.ok(content);
|
||||||
}
|
}
|
||||||
@ -153,11 +161,12 @@ public class AuthorController extends BaseController{
|
|||||||
* 更新章节内容
|
* 更新章节内容
|
||||||
*/
|
*/
|
||||||
@PostMapping("updateBookContent")
|
@PostMapping("updateBookContent")
|
||||||
public RestResult<Void> updateBookContent(Long indexId, String indexName, String content, HttpServletRequest request) {
|
public RestResult<Void> updateBookContent(Long indexId, String indexName, String content,
|
||||||
|
HttpServletRequest request) {
|
||||||
Author author = checkAuthor(request);
|
Author author = checkAuthor(request);
|
||||||
|
|
||||||
content = content.replaceAll("\\n", "<br>")
|
content = content.replaceAll("\\n", "<br>")
|
||||||
.replaceAll("\\s", " ");
|
.replaceAll("\\s", " ");
|
||||||
//更新章节内容
|
//更新章节内容
|
||||||
bookService.updateBookContent(indexId, indexName, content, author.getId());
|
bookService.updateBookContent(indexId, indexName, content, author.getId());
|
||||||
|
|
||||||
@ -168,38 +177,44 @@ public class AuthorController extends BaseController{
|
|||||||
* 修改小说封面
|
* 修改小说封面
|
||||||
*/
|
*/
|
||||||
@PostMapping("updateBookPic")
|
@PostMapping("updateBookPic")
|
||||||
public RestResult<Void> updateBookPic(@RequestParam("bookId") Long bookId,@RequestParam("bookPic") String bookPic,HttpServletRequest request) {
|
public RestResult<Void> updateBookPic(@RequestParam("bookId") Long bookId, @RequestParam("bookPic") String bookPic,
|
||||||
|
HttpServletRequest request) {
|
||||||
Author author = checkAuthor(request);
|
Author author = checkAuthor(request);
|
||||||
bookService.updateBookPic(bookId,bookPic, author.getId());
|
bookService.updateBookPic(bookId, bookPic, author.getId());
|
||||||
return RestResult.ok();
|
return RestResult.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 作家日收入统计数据分页列表查询
|
* 作家日收入统计数据分页列表查询
|
||||||
* */
|
*/
|
||||||
@GetMapping("listIncomeDailyByPage")
|
@GetMapping("listIncomeDailyByPage")
|
||||||
public RestResult<PageBean<AuthorIncomeDetail>> listIncomeDailyByPage(@RequestParam(value = "curr", defaultValue = "1") int page,
|
public RestResult<PageBean<AuthorIncomeDetail>> listIncomeDailyByPage(
|
||||||
@RequestParam(value = "limit", defaultValue = "10") int pageSize ,
|
@RequestParam(value = "curr", defaultValue = "1") int page,
|
||||||
@RequestParam(value = "bookId", defaultValue = "0") Long bookId,
|
@RequestParam(value = "limit", defaultValue = "10") int pageSize,
|
||||||
@RequestParam(value = "startTime",defaultValue = "2020-05-01") Date startTime,
|
@RequestParam(value = "bookId", defaultValue = "0") Long bookId,
|
||||||
@RequestParam(value = "endTime",defaultValue = "2030-01-01") Date endTime,
|
@RequestParam(value = "startTime", defaultValue = "2020-05-01") Date startTime,
|
||||||
HttpServletRequest request){
|
@RequestParam(value = "endTime", defaultValue = "2030-01-01") Date endTime,
|
||||||
|
HttpServletRequest request) {
|
||||||
|
|
||||||
return RestResult.ok(authorService.listIncomeDailyByPage(page,pageSize,getUserDetails(request).getId(),bookId,startTime,endTime));
|
return RestResult.ok(
|
||||||
|
authorService.listIncomeDailyByPage(page, pageSize, getUserDetails(request).getId(), bookId, startTime,
|
||||||
|
endTime));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 作家月收入统计数据分页列表查询
|
* 作家月收入统计数据分页列表查询
|
||||||
* */
|
*/
|
||||||
@GetMapping("listIncomeMonthByPage")
|
@GetMapping("listIncomeMonthByPage")
|
||||||
public RestResult<PageBean<AuthorIncome>> listIncomeMonthByPage(@RequestParam(value = "curr", defaultValue = "1") int page,
|
public RestResult<PageBean<AuthorIncome>> listIncomeMonthByPage(
|
||||||
@RequestParam(value = "limit", defaultValue = "10") int pageSize ,
|
@RequestParam(value = "curr", defaultValue = "1") int page,
|
||||||
@RequestParam(value = "bookId", defaultValue = "0") Long bookId,
|
@RequestParam(value = "limit", defaultValue = "10") int pageSize,
|
||||||
HttpServletRequest request){
|
@RequestParam(value = "bookId", defaultValue = "0") Long bookId,
|
||||||
|
HttpServletRequest request) {
|
||||||
|
|
||||||
return RestResult.ok(authorService.listIncomeMonthByPage(page,pageSize,getUserDetails(request).getId(),bookId));
|
return RestResult.ok(
|
||||||
|
authorService.listIncomeMonthByPage(page, pageSize, getUserDetails(request).getId(), bookId));
|
||||||
}
|
}
|
||||||
|
|
||||||
private Author checkAuthor(HttpServletRequest request) {
|
private Author checkAuthor(HttpServletRequest request) {
|
||||||
@ -218,22 +233,29 @@ public class AuthorController extends BaseController{
|
|||||||
throw new BusinessException(ResponseStatus.AUTHOR_STATUS_FORBIDDEN);
|
throw new BusinessException(ResponseStatus.AUTHOR_STATUS_FORBIDDEN);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return author;
|
return author;
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 查询AI生成图片
|
||||||
|
*/
|
||||||
|
@GetMapping("queryAiGenPic")
|
||||||
|
public RestResult<String> queryAiGenPic(@RequestParam("bookId") Long bookId) {
|
||||||
|
return RestResult.ok(bookService.queryAiGenPic(bookId));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* AI扩写
|
* AI扩写
|
||||||
*/
|
*/
|
||||||
@PostMapping("ai/expand")
|
@PostMapping("ai/expand")
|
||||||
public RestResult<String> expandText(@RequestParam("text") String text, @RequestParam("ratio") Double ratio) {
|
public RestResult<String> expandText(@RequestParam("text") String text, @RequestParam("ratio") Double ratio) {
|
||||||
String prompt = "请将以下文本扩写为原长度的" + ratio/100 + "倍:" + text;
|
String prompt = "请将以下文本扩写为原长度的" + ratio / 100 + "倍:" + text;
|
||||||
return RestResult.ok(chatClient.prompt()
|
return RestResult.ok(chatClient.prompt()
|
||||||
.user(prompt)
|
.user(prompt)
|
||||||
.call()
|
.call()
|
||||||
.content());
|
.content());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -241,11 +263,11 @@ public class AuthorController extends BaseController{
|
|||||||
*/
|
*/
|
||||||
@PostMapping("ai/condense")
|
@PostMapping("ai/condense")
|
||||||
public RestResult<String> condenseText(@RequestParam("text") String text, @RequestParam("ratio") Integer ratio) {
|
public RestResult<String> condenseText(@RequestParam("text") String text, @RequestParam("ratio") Integer ratio) {
|
||||||
String prompt = "请将以下文本缩写为原长度的" + 100/ratio + "分之一:" + text;
|
String prompt = "请将以下文本缩写为原长度的" + 100 / ratio + "分之一:" + text;
|
||||||
return RestResult.ok(chatClient.prompt()
|
return RestResult.ok(chatClient.prompt()
|
||||||
.user(prompt)
|
.user(prompt)
|
||||||
.call()
|
.call()
|
||||||
.content());
|
.content());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -255,9 +277,9 @@ public class AuthorController extends BaseController{
|
|||||||
public RestResult<String> continueText(@RequestParam("text") String text, @RequestParam("length") Integer length) {
|
public RestResult<String> continueText(@RequestParam("text") String text, @RequestParam("length") Integer length) {
|
||||||
String prompt = "请续写以下文本,续写长度约为" + length + "字:" + text;
|
String prompt = "请续写以下文本,续写长度约为" + length + "字:" + text;
|
||||||
return RestResult.ok(chatClient.prompt()
|
return RestResult.ok(chatClient.prompt()
|
||||||
.user(prompt)
|
.user(prompt)
|
||||||
.call()
|
.call()
|
||||||
.content());
|
.content());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -267,12 +289,57 @@ public class AuthorController extends BaseController{
|
|||||||
public RestResult<String> polishText(@RequestParam("text") String text) {
|
public RestResult<String> polishText(@RequestParam("text") String text) {
|
||||||
String prompt = "请润色优化以下文本,保持原意:" + text;
|
String prompt = "请润色优化以下文本,保持原意:" + text;
|
||||||
return RestResult.ok(chatClient.prompt()
|
return RestResult.ok(chatClient.prompt()
|
||||||
.user(prompt)
|
.user(prompt)
|
||||||
.call()
|
.call()
|
||||||
.content());
|
.content());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AI扩写
|
||||||
|
*/
|
||||||
|
@GetMapping(value = "ai/stream/expand", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
|
||||||
|
public Flux<String> streamExpandText(@RequestParam("text") String text, @RequestParam("ratio") Double ratio) {
|
||||||
|
String prompt = "请将以下文本扩写为原长度的" + ratio / 100 + "倍:" + text;
|
||||||
|
return chatClient.prompt()
|
||||||
|
.user(prompt)
|
||||||
|
.stream()
|
||||||
|
.content();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AI缩写
|
||||||
|
*/
|
||||||
|
@GetMapping(value = "ai/stream/condense", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
|
||||||
|
public Flux<String> streamCondenseText(@RequestParam("text") String text, @RequestParam("ratio") Integer ratio) {
|
||||||
|
String prompt = "请将以下文本缩写为原长度的" + 100 / ratio + "分之一:" + text;
|
||||||
|
return chatClient.prompt()
|
||||||
|
.user(prompt)
|
||||||
|
.stream()
|
||||||
|
.content();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AI续写
|
||||||
|
*/
|
||||||
|
@GetMapping(value = "ai/stream/continue", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
|
||||||
|
public Flux<String> streamContinueText(@RequestParam("text") String text, @RequestParam("length") Integer length) {
|
||||||
|
String prompt = "请续写以下文本,续写长度约为" + length + "字:" + text;
|
||||||
|
return chatClient.prompt()
|
||||||
|
.user(prompt)
|
||||||
|
.stream()
|
||||||
|
.content();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AI润色
|
||||||
|
*/
|
||||||
|
@GetMapping(value = "/ai/stream/polish", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
|
||||||
|
public Flux<String> streamPolishText(@RequestParam("text") String text) {
|
||||||
|
String prompt = "请润色优化以下文本,保持原意:" + text;
|
||||||
|
return chatClient.prompt()
|
||||||
|
.user(prompt)
|
||||||
|
.stream()
|
||||||
|
.content();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -18,16 +18,7 @@ import org.springframework.web.client.RestClient;
|
|||||||
public class AiConfig {
|
public class AiConfig {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 目的:配置自定义的 RestClientBuilder 对象
|
* 配置自定义的 RestClientBuilder 对象
|
||||||
* <p>
|
|
||||||
* 原因:Spring AI 框架的 ChatClient 内部通过 RestClient(Spring Framework 6 和 Spring Boot 3 中引入) 发起 HTTP REST 请求与远程的大模型服务进行通信,
|
|
||||||
* 如果项目中没有配置自定义的 RestClientBuilder 对象, 那么在 RestClient 的自动配置类 org.springframework.boot.autoconfigure.web.client.RestClientAutoConfiguration
|
|
||||||
* 中配置的 RestClientBuilder 对象会使用 Spring 容器中提供的 HttpMessageConverters, 由于本项目中配置了 spring.jackson.generator.write-numbers-as-strings
|
|
||||||
* = true, 所以 Spring 容器中的 HttpMessageConverters 在 RestClient 发起 HTTP REST 请求转换 Java 对象为 JSON 字符串时会自动将 Number 类型的
|
|
||||||
* Java 对象属性转换为字符串而导致请求参数错误
|
|
||||||
* <p>
|
|
||||||
* 示例:"temperature": 0.7 =》"temperature": "0.7"
|
|
||||||
* {"code":20015,"message":"The parameter is invalid. Please check again.","data":null}
|
|
||||||
*/
|
*/
|
||||||
@Bean
|
@Bean
|
||||||
public RestClient.Builder restClientBuilder() {
|
public RestClient.Builder restClientBuilder() {
|
||||||
|
@ -290,4 +290,9 @@ public interface BookService {
|
|||||||
* @param authorId
|
* @param authorId
|
||||||
*/
|
*/
|
||||||
void updateBookPic(Long bookId, String bookPic, Long authorId);
|
void updateBookPic(Long bookId, String bookPic, Long authorId);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 查询AI生成图片
|
||||||
|
*/
|
||||||
|
String queryAiGenPic(Long bookId);
|
||||||
}
|
}
|
||||||
|
@ -116,7 +116,7 @@ public class BookServiceImpl implements BookService {
|
|||||||
}
|
}
|
||||||
result = new ObjectMapper().writeValueAsString(
|
result = new ObjectMapper().writeValueAsString(
|
||||||
list.stream().collect(Collectors.groupingBy(BookSettingVO::getType)));
|
list.stream().collect(Collectors.groupingBy(BookSettingVO::getType)));
|
||||||
cacheService.set(CacheKey.INDEX_BOOK_SETTINGS_KEY, result);
|
cacheService.set(CacheKey.INDEX_BOOK_SETTINGS_KEY, result, 3600 * 24);
|
||||||
}
|
}
|
||||||
return new ObjectMapper().readValue(result, Map.class);
|
return new ObjectMapper().readValue(result, Map.class);
|
||||||
}
|
}
|
||||||
@ -559,6 +559,7 @@ public class BookServiceImpl implements BookService {
|
|||||||
.where(id, isEqualTo(book.getId()))
|
.where(id, isEqualTo(book.getId()))
|
||||||
.build()
|
.build()
|
||||||
.render(RenderingStrategies.MYBATIS3));
|
.render(RenderingStrategies.MYBATIS3));
|
||||||
|
cacheService.set(CacheKey.AI_GEN_PIC + book.getId(), picUrl, 60 * 60);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -882,5 +883,10 @@ public class BookServiceImpl implements BookService {
|
|||||||
.render(RenderingStrategies.MYBATIS3));
|
.render(RenderingStrategies.MYBATIS3));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String queryAiGenPic(Long bookId) {
|
||||||
|
return cacheService.get(CacheKey.AI_GEN_PIC + bookId);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
|
||||||
|
}
|
@ -15,7 +15,7 @@
|
|||||||
padding: 20px;
|
padding: 20px;
|
||||||
background: #fff;
|
background: #fff;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
box-shadow: 0 2px 12px 0 rgba(0,0,0,.1);
|
box-shadow: 0 2px 12px 0 rgba(0, 0, 0, .1);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* 文本域样式 */
|
/* 文本域样式 */
|
||||||
@ -124,10 +124,10 @@
|
|||||||
<li id="contentLi" style="width: 500px">
|
<li id="contentLi" style="width: 500px">
|
||||||
<div class="editor-container">
|
<div class="editor-container">
|
||||||
<div class="ai-toolbar">
|
<div class="ai-toolbar">
|
||||||
<a class="ai-link expand" data-type="expand">AI扩写</a>
|
<a class="ai-link expand" data-type="stream/expand">AI扩写</a>
|
||||||
<a class="ai-link condense" data-type="condense">AI缩写</a>
|
<a class="ai-link condense" data-type="stream/condense">AI缩写</a>
|
||||||
<a class="ai-link continue" data-type="continue">AI续写</a>
|
<a class="ai-link continue" data-type="stream/continue">AI续写</a>
|
||||||
<a class="ai-link polish" data-type="polish">AI润色</a>
|
<a class="ai-link polish" data-type="stream/polish">AI润色</a>
|
||||||
</div>
|
</div>
|
||||||
<textarea id="bookContent" name="bookContent"
|
<textarea id="bookContent" name="bookContent"
|
||||||
placeholder="请输入文本内容..."></textarea>
|
placeholder="请输入文本内容..."></textarea>
|
||||||
@ -268,7 +268,7 @@
|
|||||||
}, speed);
|
}, speed);
|
||||||
}
|
}
|
||||||
|
|
||||||
$('.ai-toolbar .ai-link').click(function(e){
|
$('.ai-toolbar .ai-link').click(function (e) {
|
||||||
e.preventDefault(); // 阻止默认链接行为
|
e.preventDefault(); // 阻止默认链接行为
|
||||||
const type = $(this).data('type');
|
const type = $(this).data('type');
|
||||||
const textarea = $('#bookContent');
|
const textarea = $('#bookContent');
|
||||||
@ -284,27 +284,27 @@
|
|||||||
|
|
||||||
// 参数配置
|
// 参数配置
|
||||||
let params = {text: selectedText};
|
let params = {text: selectedText};
|
||||||
if(type === 'expand' || type === 'condense'){
|
if (type === 'stream/expand' || type === 'stream/condense') {
|
||||||
layer.prompt({
|
layer.prompt({
|
||||||
title: '请输入比例',
|
title: '请输入比例',
|
||||||
value: 2,
|
value: 2,
|
||||||
btn: ['确定', '取消'],
|
btn: ['确定', '取消'],
|
||||||
btn2: function(){
|
btn2: function () {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
},
|
},
|
||||||
cancel: function(){
|
cancel: function () {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
}
|
}
|
||||||
}, function(value, index){
|
}, function (value, index) {
|
||||||
if(isNaN(Number(value)) || isNaN(parseFloat(value))){
|
if (isNaN(Number(value)) || isNaN(parseFloat(value))) {
|
||||||
layer.msg('请输入正确的比例');
|
layer.msg('请输入正确的比例');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if(type === 'expand' && value <= 1){
|
if (type === 'stream/expand' && value <= 1) {
|
||||||
layer.msg('请输入正确的比例');
|
layer.msg('请输入正确的比例');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if(type === 'condense' && (value <=0 || value >= 1)){
|
if (type === 'stream/condense' && (value <= 0 || value >= 1)) {
|
||||||
layer.msg('请输入正确的比例');
|
layer.msg('请输入正确的比例');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -313,19 +313,19 @@
|
|||||||
sendRequest(type, params, loading, textarea);
|
sendRequest(type, params, loading, textarea);
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}else if(type === 'continue'){
|
} else if (type === 'stream/continue') {
|
||||||
layer.prompt({
|
layer.prompt({
|
||||||
title: '请输入续写长度(字数)',
|
title: '请输入续写长度(字数)',
|
||||||
value: 200,
|
value: 200,
|
||||||
btn: ['确定', '取消'],
|
btn: ['确定', '取消'],
|
||||||
btn2: function(){
|
btn2: function () {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
},
|
},
|
||||||
cancel: function(){
|
cancel: function () {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
}
|
}
|
||||||
}, function(value, index){
|
}, function (value, index) {
|
||||||
if(!Number.isInteger(Number(value)) || value <= 0){
|
if (!Number.isInteger(Number(value)) || value <= 0) {
|
||||||
layer.msg('请输入正确的长度');
|
layer.msg('请输入正确的长度');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -339,26 +339,28 @@
|
|||||||
sendRequest(type, params, loading, textarea);
|
sendRequest(type, params, loading, textarea);
|
||||||
});
|
});
|
||||||
|
|
||||||
function sendRequest(type, params, loading, textarea){
|
function sendRequest(type, params, loading, textarea) {
|
||||||
$.ajax({
|
const url = `/author/ai/${type}?text=${encodeURIComponent(params.text)}&ratio=${params.ratio}&length=${params.length}`;
|
||||||
url: '/author/ai/' + type,
|
const eventSource = new EventSource(url);
|
||||||
type: 'POST',
|
|
||||||
data: params,
|
// 监听消息事件
|
||||||
success: function(res){
|
eventSource.onmessage = function (event) {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
// 将生成的内容追加到文本末尾
|
const data = event.data;
|
||||||
if(res.code == '200'){
|
console.log('Received data:', data);
|
||||||
const newText = "\n\n【AI生成内容】" + res.data; // 添加换行符分隔
|
|
||||||
typeWriter(textarea, newText); // 使用打字机效果
|
textarea.val(textarea.val() + data);
|
||||||
}else{
|
// 滚动到底部
|
||||||
layer.msg('AI内容生成失败,请稍后重试');
|
textarea.scrollTop(textarea[0].scrollHeight);
|
||||||
}
|
};
|
||||||
},
|
|
||||||
error: function(){
|
// 监听错误事件
|
||||||
layer.msg('请求失败,请稍后重试');
|
eventSource.onerror = function (error) {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
}
|
console.error('EventSource failed:', error);
|
||||||
});
|
eventSource.close(); // 关闭连接
|
||||||
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
|
@ -51,8 +51,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="my_r">
|
<div class="my_r">
|
||||||
<div id="noContentDiv">
|
<div id="noContentDiv">
|
||||||
<div class="tc" style="margin-top: 200px"><a href="/author/book_add.html" class="btn_red">创建作品</a>
|
<div class="tc" style="margin-top: 200px"><a href="/author/book_add.html" class="btn_red">创建作品</a></div>
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
<div class="my_bookshelf" id="hasContentDiv" style="display: none">
|
<div class="my_bookshelf" id="hasContentDiv" style="display: none">
|
||||||
@ -143,13 +142,12 @@
|
|||||||
<script src="/javascript/common.js" type="text/javascript"></script>
|
<script src="/javascript/common.js" type="text/javascript"></script>
|
||||||
|
|
||||||
<script language="javascript" type="text/javascript">
|
<script language="javascript" type="text/javascript">
|
||||||
var searchCount = 0;
|
var coverUpdateInterval;
|
||||||
var timeout;
|
|
||||||
search(1, 5);
|
search(1, 5);
|
||||||
|
|
||||||
function search(curr, limit) {
|
function search(curr, limit) {
|
||||||
searchCount++;
|
clearInterval(coverUpdateInterval);
|
||||||
clearTimeout(timeout);
|
|
||||||
$.ajax({
|
$.ajax({
|
||||||
type: "get",
|
type: "get",
|
||||||
url: "/author/listBookByPage",
|
url: "/author/listBookByPage",
|
||||||
@ -159,7 +157,25 @@
|
|||||||
if (data.code == 200) {
|
if (data.code == 200) {
|
||||||
var bookList = data.data.list;
|
var bookList = data.data.list;
|
||||||
if (bookList.length > 0) {
|
if (bookList.length > 0) {
|
||||||
var aiPicGenerating = bookList[0].picUrl == '/images/default.gif'
|
if(curr == 1 && bookList[0].picUrl == '/images/default.gif'){
|
||||||
|
coverUpdateInterval = setInterval(function(){
|
||||||
|
$.ajax({
|
||||||
|
type: "get",
|
||||||
|
url: "/author/queryAiGenPic",
|
||||||
|
data: {'bookId': bookList[0].id},
|
||||||
|
dataType: "json",
|
||||||
|
success: function (data) {
|
||||||
|
if(data.code == 200 && data.data){
|
||||||
|
$("#cover"+bookList[0].id).attr("src", data.data);
|
||||||
|
clearInterval(coverUpdateInterval);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, 3000);
|
||||||
|
setTimeout(() => {
|
||||||
|
clearInterval(coverUpdateInterval);
|
||||||
|
}, 10000);
|
||||||
|
}
|
||||||
$("#hasContentDiv").css("display", "block");
|
$("#hasContentDiv").css("display", "block");
|
||||||
$("#noContentDiv").css("display", "none");
|
$("#noContentDiv").css("display", "none");
|
||||||
var bookListHtml = "";
|
var bookListHtml = "";
|
||||||
@ -171,15 +187,12 @@
|
|||||||
" </td>\n" +*/
|
" </td>\n" +*/
|
||||||
|
|
||||||
" <td style=\"position: relative\" class=\"goread\">\n" +
|
" <td style=\"position: relative\" class=\"goread\">\n" +
|
||||||
"<input class=\"opacity\" onchange=\"picChange('" + book.id + "'," + i + ")\"\n" +
|
"<input class=\"opacity\" onchange=\"picChange('" + book.id + "')\"\n" +
|
||||||
" type=\"file\" id=\"file" + i + "\" name=\"file\"\n" +
|
" type=\"file\" id=\"file" + book.id + "\" name=\"file\"\n" +
|
||||||
" title=\"点击上传图片\"\n" +
|
" title=\"点击上传图片\"\n" +
|
||||||
" style=\"z-index: 100;cursor: pointer;left: 30px; top: 0px; width: 60px; height: 80px; opacity: 0; position: absolute; \"\n" +
|
" style=\"z-index: 100;cursor: pointer;left: 30px; top: 0px; width: 60px; height: 80px; opacity: 0; position: absolute; \"\n" +
|
||||||
" />" +
|
" />" +
|
||||||
"<img width='50' height='70' src='" + book.picUrl + "'/><br/>" +
|
"<img id=\"cover" + book.id + "\" width='50' height='70' src='" + book.picUrl + "'/><br/>" + " " + book.bookName + "</td>\n" +
|
||||||
" " + book.bookName + "</td>\n" +
|
|
||||||
|
|
||||||
|
|
||||||
" <td class=\"goread\" >"
|
" <td class=\"goread\" >"
|
||||||
+ book.catName + "</td>\n" +
|
+ book.catName + "</td>\n" +
|
||||||
|
|
||||||
@ -231,12 +244,6 @@
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
if (curr === 1 && aiPicGenerating && searchCount < 10) {
|
|
||||||
timeout = setTimeout(function () {
|
|
||||||
search(curr, limit);
|
|
||||||
}, 3000);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -288,19 +295,20 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function picChange(bookId, i) {
|
function picChange(bookId) {
|
||||||
var file = $("#file" + i).val(); //文件名称
|
var file = $("#file" + bookId).val(); //文件名称
|
||||||
if (file != "") {
|
if (file != "") {
|
||||||
if (checkPicUpload($("#file" + i)[0])) {
|
if (checkPicUpload($("#file" + bookId)[0])) {
|
||||||
|
|
||||||
$.ajaxFileUpload({
|
$.ajaxFileUpload({
|
||||||
url: "/file/picUpload", //用于文件上传的服务器端请求地址
|
url: "/file/picUpload", //用于文件上传的服务器端请求地址
|
||||||
secureuri: false, //是否需要安全协议,一般设置为false
|
secureuri: false, //是否需要安全协议,一般设置为false
|
||||||
fileElementId: "file" + i, //文件上传域的ID
|
fileElementId: "file" + bookId, //文件上传域的ID
|
||||||
dataType: "json", //返回值类型 一般设置为json
|
dataType: "json", //返回值类型 一般设置为json
|
||||||
type: "post",
|
type: "post",
|
||||||
success: function (data) { //服务器成功响应处理函数
|
success: function (data) { //服务器成功响应处理函数
|
||||||
if (data.code == 200) {
|
if (data.code == 200) {
|
||||||
|
let picUrl = data.data;
|
||||||
$.ajax({
|
$.ajax({
|
||||||
type: "POST",
|
type: "POST",
|
||||||
url: "/author/updateBookPic",
|
url: "/author/updateBookPic",
|
||||||
@ -308,17 +316,13 @@
|
|||||||
dataType: "json",
|
dataType: "json",
|
||||||
success: function (data) {
|
success: function (data) {
|
||||||
if (data.code == 200) {
|
if (data.code == 200) {
|
||||||
|
$("#cover"+bookId).attr("src", picUrl);
|
||||||
location.reload();
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
lock = false;
|
|
||||||
layer.alert(data.msg);
|
layer.alert(data.msg);
|
||||||
}
|
}
|
||||||
|
|
||||||
},
|
},
|
||||||
error: function () {
|
error: function () {
|
||||||
lock = false;
|
|
||||||
layer.alert('网络异常');
|
layer.alert('网络异常');
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
2
pom.xml
2
pom.xml
@ -5,7 +5,7 @@
|
|||||||
|
|
||||||
<groupId>com.java2nb</groupId>
|
<groupId>com.java2nb</groupId>
|
||||||
<artifactId>novel</artifactId>
|
<artifactId>novel</artifactId>
|
||||||
<version>5.1.0</version>
|
<version>5.1.5</version>
|
||||||
<modules>
|
<modules>
|
||||||
<module>novel-common</module>
|
<module>novel-common</module>
|
||||||
<module>novel-front</module>
|
<module>novel-front</module>
|
||||||
|
@ -15,7 +15,7 @@
|
|||||||
padding: 20px;
|
padding: 20px;
|
||||||
background: #fff;
|
background: #fff;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
box-shadow: 0 2px 12px 0 rgba(0,0,0,.1);
|
box-shadow: 0 2px 12px 0 rgba(0, 0, 0, .1);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* 文本域样式 */
|
/* 文本域样式 */
|
||||||
@ -124,10 +124,10 @@
|
|||||||
<li id="contentLi" style="width: 500px">
|
<li id="contentLi" style="width: 500px">
|
||||||
<div class="editor-container">
|
<div class="editor-container">
|
||||||
<div class="ai-toolbar">
|
<div class="ai-toolbar">
|
||||||
<a class="ai-link expand" data-type="expand">AI扩写</a>
|
<a class="ai-link expand" data-type="stream/expand">AI扩写</a>
|
||||||
<a class="ai-link condense" data-type="condense">AI缩写</a>
|
<a class="ai-link condense" data-type="stream/condense">AI缩写</a>
|
||||||
<a class="ai-link continue" data-type="continue">AI续写</a>
|
<a class="ai-link continue" data-type="stream/continue">AI续写</a>
|
||||||
<a class="ai-link polish" data-type="polish">AI润色</a>
|
<a class="ai-link polish" data-type="stream/polish">AI润色</a>
|
||||||
</div>
|
</div>
|
||||||
<textarea id="bookContent" name="bookContent"
|
<textarea id="bookContent" name="bookContent"
|
||||||
placeholder="请输入文本内容..."></textarea>
|
placeholder="请输入文本内容..."></textarea>
|
||||||
@ -268,7 +268,7 @@
|
|||||||
}, speed);
|
}, speed);
|
||||||
}
|
}
|
||||||
|
|
||||||
$('.ai-toolbar .ai-link').click(function(e){
|
$('.ai-toolbar .ai-link').click(function (e) {
|
||||||
e.preventDefault(); // 阻止默认链接行为
|
e.preventDefault(); // 阻止默认链接行为
|
||||||
const type = $(this).data('type');
|
const type = $(this).data('type');
|
||||||
const textarea = $('#bookContent');
|
const textarea = $('#bookContent');
|
||||||
@ -284,27 +284,27 @@
|
|||||||
|
|
||||||
// 参数配置
|
// 参数配置
|
||||||
let params = {text: selectedText};
|
let params = {text: selectedText};
|
||||||
if(type === 'expand' || type === 'condense'){
|
if (type === 'stream/expand' || type === 'stream/condense') {
|
||||||
layer.prompt({
|
layer.prompt({
|
||||||
title: '请输入比例',
|
title: '请输入比例',
|
||||||
value: 2,
|
value: 2,
|
||||||
btn: ['确定', '取消'],
|
btn: ['确定', '取消'],
|
||||||
btn2: function(){
|
btn2: function () {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
},
|
},
|
||||||
cancel: function(){
|
cancel: function () {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
}
|
}
|
||||||
}, function(value, index){
|
}, function (value, index) {
|
||||||
if(isNaN(Number(value)) || isNaN(parseFloat(value))){
|
if (isNaN(Number(value)) || isNaN(parseFloat(value))) {
|
||||||
layer.msg('请输入正确的比例');
|
layer.msg('请输入正确的比例');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if(type === 'expand' && value <= 1){
|
if (type === 'stream/expand' && value <= 1) {
|
||||||
layer.msg('请输入正确的比例');
|
layer.msg('请输入正确的比例');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if(type === 'condense' && (value <=0 || value >= 1)){
|
if (type === 'stream/condense' && (value <= 0 || value >= 1)) {
|
||||||
layer.msg('请输入正确的比例');
|
layer.msg('请输入正确的比例');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -313,19 +313,19 @@
|
|||||||
sendRequest(type, params, loading, textarea);
|
sendRequest(type, params, loading, textarea);
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}else if(type === 'continue'){
|
} else if (type === 'stream/continue') {
|
||||||
layer.prompt({
|
layer.prompt({
|
||||||
title: '请输入续写长度(字数)',
|
title: '请输入续写长度(字数)',
|
||||||
value: 200,
|
value: 200,
|
||||||
btn: ['确定', '取消'],
|
btn: ['确定', '取消'],
|
||||||
btn2: function(){
|
btn2: function () {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
},
|
},
|
||||||
cancel: function(){
|
cancel: function () {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
}
|
}
|
||||||
}, function(value, index){
|
}, function (value, index) {
|
||||||
if(!Number.isInteger(Number(value)) || value <= 0){
|
if (!Number.isInteger(Number(value)) || value <= 0) {
|
||||||
layer.msg('请输入正确的长度');
|
layer.msg('请输入正确的长度');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -339,26 +339,28 @@
|
|||||||
sendRequest(type, params, loading, textarea);
|
sendRequest(type, params, loading, textarea);
|
||||||
});
|
});
|
||||||
|
|
||||||
function sendRequest(type, params, loading, textarea){
|
function sendRequest(type, params, loading, textarea) {
|
||||||
$.ajax({
|
const url = `/author/ai/${type}?text=${encodeURIComponent(params.text)}&ratio=${params.ratio}&length=${params.length}`;
|
||||||
url: '/author/ai/' + type,
|
const eventSource = new EventSource(url);
|
||||||
type: 'POST',
|
|
||||||
data: params,
|
// 监听消息事件
|
||||||
success: function(res){
|
eventSource.onmessage = function (event) {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
// 将生成的内容追加到文本末尾
|
const data = event.data;
|
||||||
if(res.code == '200'){
|
console.log('Received data:', data);
|
||||||
const newText = "\n\n【AI生成内容】" + res.data; // 添加换行符分隔
|
|
||||||
typeWriter(textarea, newText); // 使用打字机效果
|
textarea.val(textarea.val() + data);
|
||||||
}else{
|
// 滚动到底部
|
||||||
layer.msg('AI内容生成失败,请稍后重试');
|
textarea.scrollTop(textarea[0].scrollHeight);
|
||||||
}
|
};
|
||||||
},
|
|
||||||
error: function(){
|
// 监听错误事件
|
||||||
layer.msg('请求失败,请稍后重试');
|
eventSource.onerror = function (error) {
|
||||||
layer.close(loading);
|
layer.close(loading);
|
||||||
}
|
console.error('EventSource failed:', error);
|
||||||
});
|
eventSource.close(); // 关闭连接
|
||||||
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
|
Reference in New Issue
Block a user