mirror of
https://github.com/201206030/novel-plus.git
synced 2025-06-30 23:06:38 +00:00
Compare commits
18 Commits
v5.1.1
...
develop_xx
Author | SHA1 | Date | |
---|---|---|---|
4693c7ffae | |||
efb136e3be | |||
7955db0e3c | |||
60dc28c5ed | |||
1534220f0c | |||
0830f6ffeb | |||
adc83db64e | |||
9c11f22816 | |||
24abe7714f | |||
a9fc80eba1 | |||
32541a7cb6 | |||
42bcecc304 | |||
a07643bde0 | |||
1f53b56bd6 | |||
2c86cb9a7d | |||
a4d6272a4f | |||
55d5deea74 | |||
4f474b91a8 |
16
README.md
16
README.md
@ -1,5 +1,4 @@
|
||||
<p align="center">
|
||||
<a href="https://www.swiftproxy.net/?code=T2WV1VT50"><img src="https://xxyopen.com/images/ad1.png" alt="AD" ></a>
|
||||
<a href="https://cloud.tencent.com/act/cps/redirect?redirect=2446&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console"><img src="https://youdoc.github.io/img/tencent.jpg" alt="AD" ></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
@ -10,7 +9,7 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
👉 <a href='https://novel.xxyopen.com'>官网</a> | 👉 <a href='https://www.bilibili.com/video/BV1Zo4y187Mi'>项目演示</a> | 👉 <a href='https://docs.xxyopen.com/course/novelplus/1.html'>安装教程</a>
|
||||
👉 <a href='https://novel.xxyopen.com'>官网</a> | 👉 <a href='http://117.72.165.13:8888'>演示站点</a> | 👉 <a href='https://docs.xxyopen.com/course/novelplus/1.html'>安装教程</a>
|
||||
</p>
|
||||
|
||||
## 项目介绍
|
||||
@ -22,8 +21,8 @@ TXT 文本存储)、阅读主题切换、多爬虫源自动采集和更新数
|
||||
## 项目地址
|
||||
|
||||
- 学习版:[GitHub](https://github.com/201206030/novel) | [码云](https://gitee.com/novel_dev_team/novel)
|
||||
| [保姆级教程](https://docs.xxyopen.com)
|
||||
- **应用版**:[GitHub](https://github.com/201206030/novel-plus) | [码云](https://gitee.com/novel_dev_team/novel-plus)
|
||||
| [保姆级教程](https://docs.xxyopen.com)
|
||||
- **应用版**:[GitHub](https://github.com/201206030/novel-plus) | [码云](https://gitee.com/novel_dev_team/novel-plus) | [演示站点](http://117.72.165.13:8888)
|
||||
- 微服务版:[GitHub](https://github.com/201206030/novel-cloud) | [码云](https://gitee.com/novel_dev_team/novel-cloud)
|
||||
|
||||
## 项目结构
|
||||
@ -78,14 +77,11 @@ novel-plus 5.x 已集成 Spring 官方最新发布的 Spring AI 框架,并推
|
||||
1. v5.0.0 版本在小说章节发布页面的文本编辑器中集成了多项智能编辑功能,包括 AI 扩写、缩写、续写及文本润色等。这些功能的设计灵感来源于百家号文章编辑器中的 AI 助手。
|
||||
2. v5.1.0 版本在小说发布页面,新增 AI 生成封面图功能。若作家未上传自定义封面图,系统将根据小说信息自动生成封面图。
|
||||
|
||||
目前,AI 功能仍处于实验阶段,仅实现了基础的核心功能。我们非常重视用户的实际使用体验和反馈,未来将根据用户需求和使用情况,持续优化和调整该功能。如果用户反馈积极,我们计划进一步开发更高级的
|
||||
AI 功能,例如自动生成有声小说、智能情节推荐等,以全面提升 novel-plus 的创作能力和用户体验。
|
||||
目前,AI 功能仍处于实验阶段,仅实现了基础的核心功能。我们非常重视用户的实际使用体验和反馈,未来将根据用户需求和使用情况,持续优化和调整该功能。如果用户反馈积极,我们计划进一步开发更高级的 AI 功能,例如自动生成有声小说、智能情节推荐等,以全面提升 novel-plus 的创作能力和用户体验。
|
||||
|
||||
我们将持续关注 AI 技术的发展,并致力于将其与小说创作场景深度融合,为用户带来更智能、更便捷的创作工具。
|
||||
|
||||
由于 DeepSeek 官方 API 目前不可用,novel-plus 项目默认使用的是第三方[硅基流动](https://cloud.siliconflow.cn/i/DOgMRH9S)
|
||||
提供的 API,采用的 AI 模型有对话模型`deepseek-ai/DeepSeek-R1-Distill-Llama-8B`(DeepSeek-R1 的蒸馏版本,免费使用)和生图模型`Kwai-Kolors/Kolors`(快手 Kolors 团队开发的文本到图像生成模型,免费使用)。只需注册一个硅基流动账号,创建一个
|
||||
API 密钥,并将其添加到 novel-plus 项目 novel-front 模块的 yaml 配置文件中,即可体验 novel-plus 项目的 AI 写作功能。
|
||||
novel-plus 项目默认使用的是第三方大模型服务平台[硅基流动](https://cloud.siliconflow.cn/i/DOgMRH9S)提供的 API(兼容 OpenAI 的相关接口,可直接通过 Spring AI 框架调用),采用的 AI 模型有对话模型`deepseek-ai/DeepSeek-R1-Distill-Llama-8B`(DeepSeek-R1 的蒸馏版本,免费使用)和生图模型`Kwai-Kolors/Kolors`(快手 Kolors 团队开发的文本到图像生成模型,免费使用)。只需注册一个硅基流动账号,创建一个 API 密钥,并将其添加到 novel-plus 项目 novel-front 模块的 yaml 配置文件中,即可体验 novel-plus 项目的 AI 写作功能。
|
||||
|
||||
```yaml
|
||||
spring:
|
||||
@ -105,7 +101,7 @@ spring:
|
||||
model: deepseek-ai/DeepSeek-R1-Distill-Llama-8B
|
||||
```
|
||||
|
||||
> ⚠️ novel-plus 项目默认使用的都是免费 AI 模型,生成效果有限。如果对生成内容有更高的要求,建议选用付费的 AI 模型。
|
||||
⚠️ novel-plus 项目默认使用的都是免费 AI 模型,生成效果有限。如果对生成内容有更高的要求,建议选用付费的 AI 模型。
|
||||
|
||||
## 增值服务
|
||||
|
||||
|
3
doc/sql/20250630.sql
Normal file
3
doc/sql/20250630.sql
Normal file
@ -0,0 +1,3 @@
|
||||
alter table book_comment add column location varchar(50) DEFAULT NULL COMMENT '地理位置' after comment_content ;
|
||||
|
||||
|
@ -3153,4 +3153,9 @@ where menu_id = 104;
|
||||
|
||||
delete
|
||||
from sys_menu
|
||||
where menu_id = 57;
|
||||
where menu_id = 57;
|
||||
|
||||
|
||||
alter table book_comment add column location varchar(50) DEFAULT NULL COMMENT '地理位置' after comment_content ;
|
||||
|
||||
|
||||
|
@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.java2nb</groupId>
|
||||
<artifactId>novel-admin</artifactId>
|
||||
<version>5.1.1</version>
|
||||
<version>5.1.5</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>novel-admin</name>
|
||||
|
@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>novel</artifactId>
|
||||
<groupId>com.java2nb</groupId>
|
||||
<version>5.1.1</version>
|
||||
<version>5.1.5</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
@ -41,11 +41,6 @@ public interface CacheKey {
|
||||
* */
|
||||
String TEMPLATE_DIR_KEY = "templateDirKey";;
|
||||
|
||||
/**
|
||||
* 正在运行的爬虫线程存储KEY前缀
|
||||
* */
|
||||
String RUNNING_CRAWL_THREAD_KEY_PREFIX = "runningCrawlTreadDataKeyPrefix";
|
||||
|
||||
/**
|
||||
* 上一次搜索引擎更新的时间
|
||||
* */
|
||||
|
@ -5,8 +5,8 @@ import org.springframework.http.*;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* @author Administrator
|
||||
@ -16,7 +16,7 @@ public class HttpUtil {
|
||||
|
||||
private static final String DEFAULT_CHARSET = "utf-8";
|
||||
|
||||
private static final Map<String, RestTemplate> REST_TEMPLATE_MAP = new HashMap<>();
|
||||
private static final Map<String, RestTemplate> REST_TEMPLATE_MAP = new ConcurrentHashMap<>();
|
||||
|
||||
public static String getByHttpClientWithChrome(String url, String charset) {
|
||||
log.debug("Get url:{}", url);
|
||||
|
@ -1,11 +1,21 @@
|
||||
package com.java2nb.novel.core.utils;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
|
||||
@Slf4j
|
||||
public class IpUtil {
|
||||
|
||||
/**
|
||||
* 获取真实IP
|
||||
*
|
||||
* @param request 请求体
|
||||
* @return 真实IP
|
||||
*/
|
||||
@ -31,4 +41,27 @@ public class IpUtil {
|
||||
}
|
||||
return ip;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取本机公网IP
|
||||
*/
|
||||
public static String getPublicIP() {
|
||||
try {
|
||||
HttpClient client = HttpClient.newHttpClient();
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create("https://httpbin.org/ip"))
|
||||
.GET()
|
||||
.timeout(Duration.ofSeconds(5))
|
||||
.build();
|
||||
|
||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
|
||||
if (response.statusCode() == 200) {
|
||||
return new ObjectMapper().readTree(response.body()).get("origin").asText();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("获取本机公网IP异常", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
@ -14,6 +14,9 @@ public class BookComment {
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private String commentContent;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private String location;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
private Integer replyCount;
|
||||
|
||||
@ -56,6 +59,16 @@ public class BookComment {
|
||||
this.commentContent = commentContent == null ? null : commentContent.trim();
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public String getLocation() {
|
||||
return location;
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public void setLocation(String location) {
|
||||
this.location = location == null ? null : location.trim();
|
||||
}
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public Integer getReplyCount() {
|
||||
return replyCount;
|
||||
|
@ -19,6 +19,9 @@ public final class BookCommentDynamicSqlSupport {
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<String> commentContent = bookComment.commentContent;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<String> location = bookComment.location;
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
public static final SqlColumn<Integer> replyCount = bookComment.replyCount;
|
||||
|
||||
@ -39,6 +42,8 @@ public final class BookCommentDynamicSqlSupport {
|
||||
|
||||
public final SqlColumn<String> commentContent = column("comment_content", JDBCType.VARCHAR);
|
||||
|
||||
public final SqlColumn<String> location = column("location", JDBCType.VARCHAR);
|
||||
|
||||
public final SqlColumn<Integer> replyCount = column("reply_count", JDBCType.INTEGER);
|
||||
|
||||
public final SqlColumn<Byte> auditStatus = column("audit_status", JDBCType.TINYINT);
|
||||
|
@ -29,7 +29,7 @@ import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
|
||||
@Mapper
|
||||
public interface BookCommentMapper {
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
BasicColumn[] selectList = BasicColumn.columnList(id, bookId, commentContent, replyCount, auditStatus, createTime, createUserId);
|
||||
BasicColumn[] selectList = BasicColumn.columnList(id, bookId, commentContent, location, replyCount, auditStatus, createTime, createUserId);
|
||||
|
||||
@Generated("org.mybatis.generator.api.MyBatisGenerator")
|
||||
@SelectProvider(type=SqlProviderAdapter.class, method="select")
|
||||
@ -58,6 +58,7 @@ public interface BookCommentMapper {
|
||||
@Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
|
||||
@Result(column="book_id", property="bookId", jdbcType=JdbcType.BIGINT),
|
||||
@Result(column="comment_content", property="commentContent", jdbcType=JdbcType.VARCHAR),
|
||||
@Result(column="location", property="location", jdbcType=JdbcType.VARCHAR),
|
||||
@Result(column="reply_count", property="replyCount", jdbcType=JdbcType.INTEGER),
|
||||
@Result(column="audit_status", property="auditStatus", jdbcType=JdbcType.TINYINT),
|
||||
@Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP),
|
||||
@ -92,6 +93,7 @@ public interface BookCommentMapper {
|
||||
c.map(id).toProperty("id")
|
||||
.map(bookId).toProperty("bookId")
|
||||
.map(commentContent).toProperty("commentContent")
|
||||
.map(location).toProperty("location")
|
||||
.map(replyCount).toProperty("replyCount")
|
||||
.map(auditStatus).toProperty("auditStatus")
|
||||
.map(createTime).toProperty("createTime")
|
||||
@ -105,6 +107,7 @@ public interface BookCommentMapper {
|
||||
c.map(id).toProperty("id")
|
||||
.map(bookId).toProperty("bookId")
|
||||
.map(commentContent).toProperty("commentContent")
|
||||
.map(location).toProperty("location")
|
||||
.map(replyCount).toProperty("replyCount")
|
||||
.map(auditStatus).toProperty("auditStatus")
|
||||
.map(createTime).toProperty("createTime")
|
||||
@ -118,6 +121,7 @@ public interface BookCommentMapper {
|
||||
c.map(id).toPropertyWhenPresent("id", record::getId)
|
||||
.map(bookId).toPropertyWhenPresent("bookId", record::getBookId)
|
||||
.map(commentContent).toPropertyWhenPresent("commentContent", record::getCommentContent)
|
||||
.map(location).toPropertyWhenPresent("location", record::getLocation)
|
||||
.map(replyCount).toPropertyWhenPresent("replyCount", record::getReplyCount)
|
||||
.map(auditStatus).toPropertyWhenPresent("auditStatus", record::getAuditStatus)
|
||||
.map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime)
|
||||
@ -157,6 +161,7 @@ public interface BookCommentMapper {
|
||||
return dsl.set(id).equalTo(record::getId)
|
||||
.set(bookId).equalTo(record::getBookId)
|
||||
.set(commentContent).equalTo(record::getCommentContent)
|
||||
.set(location).equalTo(record::getLocation)
|
||||
.set(replyCount).equalTo(record::getReplyCount)
|
||||
.set(auditStatus).equalTo(record::getAuditStatus)
|
||||
.set(createTime).equalTo(record::getCreateTime)
|
||||
@ -168,6 +173,7 @@ public interface BookCommentMapper {
|
||||
return dsl.set(id).equalToWhenPresent(record::getId)
|
||||
.set(bookId).equalToWhenPresent(record::getBookId)
|
||||
.set(commentContent).equalToWhenPresent(record::getCommentContent)
|
||||
.set(location).equalToWhenPresent(record::getLocation)
|
||||
.set(replyCount).equalToWhenPresent(record::getReplyCount)
|
||||
.set(auditStatus).equalToWhenPresent(record::getAuditStatus)
|
||||
.set(createTime).equalToWhenPresent(record::getCreateTime)
|
||||
@ -179,6 +185,7 @@ public interface BookCommentMapper {
|
||||
return update(c ->
|
||||
c.set(bookId).equalTo(record::getBookId)
|
||||
.set(commentContent).equalTo(record::getCommentContent)
|
||||
.set(location).equalTo(record::getLocation)
|
||||
.set(replyCount).equalTo(record::getReplyCount)
|
||||
.set(auditStatus).equalTo(record::getAuditStatus)
|
||||
.set(createTime).equalTo(record::getCreateTime)
|
||||
@ -192,6 +199,7 @@ public interface BookCommentMapper {
|
||||
return update(c ->
|
||||
c.set(bookId).equalToWhenPresent(record::getBookId)
|
||||
.set(commentContent).equalToWhenPresent(record::getCommentContent)
|
||||
.set(location).equalToWhenPresent(record::getLocation)
|
||||
.set(replyCount).equalToWhenPresent(record::getReplyCount)
|
||||
.set(auditStatus).equalToWhenPresent(record::getAuditStatus)
|
||||
.set(createTime).equalToWhenPresent(record::getCreateTime)
|
||||
|
@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>novel</artifactId>
|
||||
<groupId>com.java2nb</groupId>
|
||||
<version>5.1.1</version>
|
||||
<version>5.1.5</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
@ -7,6 +7,6 @@ import com.java2nb.novel.entity.Book;
|
||||
* */
|
||||
public interface CrawlBookHandler {
|
||||
|
||||
void handle(Book book);
|
||||
void handle(Book book) throws InterruptedException;
|
||||
|
||||
}
|
||||
|
@ -10,9 +10,11 @@ import com.java2nb.novel.utils.CrawlHttpClient;
|
||||
import io.github.xxyopen.util.IdWorker;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
@ -26,6 +28,7 @@ import java.util.regex.Pattern;
|
||||
*
|
||||
* @author Administrator
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class CrawlParser {
|
||||
@ -34,8 +37,8 @@ public class CrawlParser {
|
||||
|
||||
private final CrawlHttpClient crawlHttpClient;
|
||||
|
||||
@SneakyThrows
|
||||
public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler) {
|
||||
public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler)
|
||||
throws InterruptedException {
|
||||
Book book = new Book();
|
||||
String bookDetailUrl = ruleBean.getBookDetailUrl().replace("{bookId}", bookId);
|
||||
String bookDetailHtml = crawlHttpClient.get(bookDetailUrl, ruleBean.getCharset());
|
||||
@ -97,6 +100,22 @@ public class CrawlParser {
|
||||
.replaceAll("<p>\\s*</p>", "")
|
||||
.replaceAll("<p>", "")
|
||||
.replaceAll("</p>", "<br/>");
|
||||
// 小说简介过滤
|
||||
String filterDesc = ruleBean.getFilterDesc();
|
||||
if (StringUtils.isNotBlank(filterDesc)) {
|
||||
String[] filterRules = filterDesc.replace("\r\n", "\n").split("\n");
|
||||
for (String filterRule : filterRules) {
|
||||
if (StringUtils.isNotBlank(filterRule)) {
|
||||
desc = desc.replaceAll(filterRule, "");
|
||||
}
|
||||
}
|
||||
}
|
||||
// 去除小说简介前后空格
|
||||
desc = desc.trim();
|
||||
// 去除小说简介末尾冗余的小说名
|
||||
if (desc.endsWith(bookName)) {
|
||||
desc = desc.substring(0, desc.length() - bookName.length());
|
||||
}
|
||||
//设置书籍简介
|
||||
book.setBookDesc(desc);
|
||||
if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) {
|
||||
@ -120,8 +139,12 @@ public class CrawlParser {
|
||||
if (isFindUpdateTime) {
|
||||
String updateTime = updateTimeMatch.group(1);
|
||||
//设置更新时间
|
||||
book.setLastIndexUpdateTime(
|
||||
new SimpleDateFormat(ruleBean.getUpadateTimeFormatPatten()).parse(updateTime));
|
||||
try {
|
||||
book.setLastIndexUpdateTime(
|
||||
new SimpleDateFormat(ruleBean.getUpadateTimeFormatPatten()).parse(updateTime));
|
||||
} catch (ParseException e) {
|
||||
log.error("解析最新章节更新时间出错", e);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -144,7 +167,7 @@ public class CrawlParser {
|
||||
}
|
||||
|
||||
public boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean,
|
||||
Map<Integer, BookIndex> existBookIndexMap, CrawlBookChapterHandler handler) {
|
||||
Map<Integer, BookIndex> existBookIndexMap, CrawlBookChapterHandler handler) throws InterruptedException {
|
||||
|
||||
Date currentDate = new Date();
|
||||
|
||||
@ -231,6 +254,8 @@ public class CrawlParser {
|
||||
}
|
||||
}
|
||||
}
|
||||
// 去除小说内容末尾的所有换行
|
||||
content = removeTrailingBrTags(content);
|
||||
//插入章节目录和章节内容
|
||||
BookIndex bookIndex = new BookIndex();
|
||||
bookIndex.setIndexName(indexName);
|
||||
@ -307,4 +332,12 @@ public class CrawlParser {
|
||||
return false;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除字符串末尾的所有 <br> 类似标签(允许各种空格)
|
||||
*/
|
||||
public static String removeTrailingBrTags(String str) {
|
||||
return str.replaceAll("(?i)(?:\\s*<\\s*br\\s*/?\\s*>)++(?:\\s|\\u3000)*$", "");
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -45,6 +45,7 @@ public class RuleBean {
|
||||
private String visitCountPatten;
|
||||
private String descStart;
|
||||
private String descEnd;
|
||||
private String filterDesc;
|
||||
private String upadateTimePatten;
|
||||
private String upadateTimeFormatPatten;
|
||||
private String bookIndexUrl;
|
||||
|
@ -74,10 +74,8 @@ public class StarterListener implements ServletContextInitializer {
|
||||
needUpdateBook.getId());
|
||||
//解析章节目录
|
||||
crawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(), book,
|
||||
ruleBean, existBookIndexMap, chapter -> {
|
||||
bookService.updateBookAndIndexAndContent(book, chapter.getBookIndexList(),
|
||||
chapter.getBookContentList(), existBookIndexMap);
|
||||
});
|
||||
ruleBean, existBookIndexMap, chapter -> bookService.updateBookAndIndexAndContent(book, chapter.getBookIndexList(),
|
||||
chapter.getBookContentList(), existBookIndexMap));
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
|
@ -1,61 +0,0 @@
|
||||
package com.java2nb.novel.core.schedule;
|
||||
|
||||
|
||||
import com.java2nb.novel.core.cache.CacheKey;
|
||||
import com.java2nb.novel.core.cache.CacheService;
|
||||
import com.java2nb.novel.entity.CrawlSource;
|
||||
import com.java2nb.novel.service.CrawlService;
|
||||
import io.github.xxyopen.util.ThreadUtil;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 爬虫线程监控器,监控执行完成的爬虫源,并修改状态
|
||||
*
|
||||
* @author Administrator
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CrawlThreadMonitor {
|
||||
|
||||
private final CacheService cacheService;
|
||||
|
||||
private final CrawlService crawlService;
|
||||
|
||||
@Scheduled(fixedRate = 1000 * 60 * 5)
|
||||
public void monitor() {
|
||||
|
||||
//查询需要监控的正在运行的爬虫源
|
||||
List<CrawlSource> sources = crawlService.queryCrawlSourceByStatus((byte) 1);
|
||||
|
||||
for (CrawlSource source : sources) {
|
||||
Set<Long> runningCrawlThreadIds = (Set<Long>) cacheService.getObject(CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + source.getId());
|
||||
boolean sourceStop = true;
|
||||
if (runningCrawlThreadIds != null) {
|
||||
for (Long threadId : runningCrawlThreadIds) {
|
||||
Thread thread = ThreadUtil.findThread(threadId);
|
||||
|
||||
if (thread != null && thread.isAlive()) {
|
||||
//有活跃线程,说明该爬虫源正在运行,数据库中状态正确,不需要修改
|
||||
sourceStop = false;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceStop) {
|
||||
crawlService.updateCrawlSourceStatus(source.getId(), (byte) 0);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
@ -53,7 +53,7 @@ public interface CrawlService {
|
||||
* @param ruleBean 采集规则\
|
||||
* @return true:成功,false:失败
|
||||
* */
|
||||
boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId);
|
||||
boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) throws InterruptedException;
|
||||
|
||||
/**
|
||||
* 根据爬虫状态查询爬虫源集合
|
||||
|
@ -34,6 +34,7 @@ import org.mybatis.dynamic.sql.render.RenderingStrategies;
|
||||
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.regex.Matcher;
|
||||
@ -65,6 +66,10 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
|
||||
private final CrawlHttpClient crawlHttpClient;
|
||||
|
||||
private final Map<Integer, Byte> crawlSourceStatusMap = new HashMap<>();
|
||||
|
||||
private final Map<Integer, Set<Long>> runningCrawlThread = new HashMap<>();
|
||||
|
||||
|
||||
@Override
|
||||
public void addCrawlSource(CrawlSource source) {
|
||||
@ -103,6 +108,8 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
.build()
|
||||
.render(RenderingStrategies.MYBATIS3);
|
||||
List<CrawlSource> crawlSources = crawlSourceMapper.selectMany(render);
|
||||
crawlSources.forEach(crawlSource -> crawlSource.setSourceStatus(
|
||||
Optional.ofNullable(crawlSourceStatusMap.get(crawlSource.getId())).orElse((byte) 0)));
|
||||
PageBean<CrawlSource> pageBean = PageBuilder.build(crawlSources);
|
||||
pageBean.setList(BeanUtil.copyList(crawlSources, CrawlSourceVO.class));
|
||||
return pageBean;
|
||||
@ -112,14 +119,13 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
@Override
|
||||
public void openOrCloseCrawl(Integer sourceId, Byte sourceStatus) {
|
||||
|
||||
//判断是开启还是关闭,如果是关闭,则修改数据库状态后获取该爬虫正在运行的线程集合并全部停止
|
||||
//如果是开启,先查询数据库中状态,判断该爬虫源是否还在运行,如果在运行,则忽略,
|
||||
// 如果没有则修改数据库状态,并启动线程爬取小说数据加入到runningCrawlThread中
|
||||
// 判断是开启还是关闭,如果是关闭,则获取该爬虫源正在运行的线程集合并全部中断
|
||||
// 如果是开启,先判断该爬虫源是否还在运行,如果在运行,则忽略,如果没有运行则启动线程爬取小说数据并加入到runningCrawlThread中
|
||||
// 最后,保存爬虫源状态
|
||||
if (sourceStatus == (byte) 0) {
|
||||
//关闭,直接修改数据库状态,并直接修改数据库状态后获取该爬虫正在运行的线程集合全部停止
|
||||
SpringUtil.getBean(CrawlService.class).updateCrawlSourceStatus(sourceId, sourceStatus);
|
||||
Set<Long> runningCrawlThreadId = (Set<Long>) cacheService.getObject(
|
||||
CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + sourceId);
|
||||
// 关闭
|
||||
// 将该爬虫源正在运行的线程集合全部停止
|
||||
Set<Long> runningCrawlThreadId = runningCrawlThread.get(sourceId);
|
||||
if (runningCrawlThreadId != null) {
|
||||
for (Long ThreadId : runningCrawlThreadId) {
|
||||
Thread thread = ThreadUtil.findThread(ThreadId);
|
||||
@ -131,16 +137,13 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
|
||||
|
||||
} else {
|
||||
//开启
|
||||
//查询爬虫源状态和规则
|
||||
CrawlSource source = queryCrawlSource(sourceId);
|
||||
Byte realSourceStatus = source.getSourceStatus();
|
||||
|
||||
// 开启
|
||||
Byte realSourceStatus = Optional.ofNullable(crawlSourceStatusMap.get(sourceId)).orElse((byte) 0);
|
||||
if (realSourceStatus == (byte) 0) {
|
||||
//该爬虫源已经停止运行了,修改数据库状态,并启动线程爬取小说数据加入到runningCrawlThread中
|
||||
SpringUtil.getBean(CrawlService.class).updateCrawlSourceStatus(sourceId, sourceStatus);
|
||||
// 查询爬虫源规则
|
||||
CrawlSource source = queryCrawlSource(sourceId);
|
||||
//该爬虫源已经停止运行了,启动线程爬取小说数据并将线程加入到runningCrawlThread中
|
||||
RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class);
|
||||
|
||||
Set<Long> threadIds = new HashSet<>();
|
||||
//按分类开始爬虫解析任务
|
||||
for (int i = 1; i < 8; i++) {
|
||||
@ -149,16 +152,15 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
thread.start();
|
||||
//thread加入到监控缓存中
|
||||
threadIds.add(thread.getId());
|
||||
|
||||
}
|
||||
cacheService.setObject(CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + sourceId, threadIds);
|
||||
|
||||
|
||||
runningCrawlThread.put(sourceId, threadIds);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
// 保存爬虫源状态
|
||||
crawlSourceStatusMap.put(sourceId, sourceStatus);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -248,6 +250,11 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
@Override
|
||||
public void parseBookList(int catId, RuleBean ruleBean, Integer sourceId) {
|
||||
|
||||
String catIdRule = ruleBean.getCatIdRule().get("catId" + catId);
|
||||
if (StringUtils.isBlank(catIdRule)) {
|
||||
return;
|
||||
}
|
||||
|
||||
//当前页码1
|
||||
int page = 1;
|
||||
int totalPage = page;
|
||||
@ -255,73 +262,91 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
while (page <= totalPage) {
|
||||
|
||||
try {
|
||||
String catIdRule = ruleBean.getCatIdRule().get("catId" + catId);
|
||||
if (StringUtils.isNotBlank(catIdRule)) {
|
||||
String catBookListUrl = "";
|
||||
if (StringUtils.isNotBlank(ruleBean.getBookListUrl())) {
|
||||
// 兼容老规则
|
||||
// 拼接分类URL
|
||||
catBookListUrl = ruleBean.getBookListUrl()
|
||||
.replace("{catId}", catIdRule)
|
||||
.replace("{page}", page + "");
|
||||
} else {
|
||||
// 新规则
|
||||
// 拼接分类URL
|
||||
catBookListUrl = catIdRule.replace("{page}", page + "");
|
||||
}
|
||||
log.info("catBookListUrl:{}", catBookListUrl);
|
||||
String catBookListUrl;
|
||||
if (StringUtils.isNotBlank(ruleBean.getBookListUrl())) {
|
||||
// 兼容老规则
|
||||
// 拼接分类URL
|
||||
catBookListUrl = ruleBean.getBookListUrl()
|
||||
.replace("{catId}", catIdRule)
|
||||
.replace("{page}", page + "");
|
||||
} else {
|
||||
// 新规则
|
||||
// 拼接分类URL
|
||||
catBookListUrl = catIdRule.replace("{page}", page + "");
|
||||
}
|
||||
log.info("catBookListUrl:{}", catBookListUrl);
|
||||
|
||||
String bookListHtml = crawlHttpClient.get(catBookListUrl, ruleBean.getCharset());
|
||||
if (bookListHtml != null) {
|
||||
Pattern bookIdPatten = Pattern.compile(ruleBean.getBookIdPatten());
|
||||
Matcher bookIdMatcher = bookIdPatten.matcher(bookListHtml);
|
||||
boolean isFindBookId = bookIdMatcher.find();
|
||||
while (isFindBookId) {
|
||||
try {
|
||||
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||
//捕获中断异常InterruptedException来退出线程。
|
||||
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
return;
|
||||
}
|
||||
|
||||
String bookId = bookIdMatcher.group(1);
|
||||
parseBookAndSave(catId, ruleBean, sourceId, bookId);
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
String bookListHtml = crawlHttpClient.get(catBookListUrl, ruleBean.getCharset());
|
||||
if (bookListHtml != null) {
|
||||
Pattern bookIdPatten = Pattern.compile(ruleBean.getBookIdPatten());
|
||||
Matcher bookIdMatcher = bookIdPatten.matcher(bookListHtml);
|
||||
boolean isFindBookId = bookIdMatcher.find();
|
||||
while (isFindBookId) {
|
||||
try {
|
||||
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||
//捕获中断异常InterruptedException来退出线程。
|
||||
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
return;
|
||||
}
|
||||
|
||||
isFindBookId = bookIdMatcher.find();
|
||||
String bookId = bookIdMatcher.group(1);
|
||||
parseBookAndSave(catId, ruleBean, sourceId, bookId);
|
||||
} catch (InterruptedException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||
//捕获中断异常InterruptedException来退出线程。
|
||||
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
Pattern totalPagePatten = Pattern.compile(ruleBean.getTotalPagePatten());
|
||||
Matcher totalPageMatcher = totalPagePatten.matcher(bookListHtml);
|
||||
boolean isFindTotalPage = totalPageMatcher.find();
|
||||
if (isFindTotalPage) {
|
||||
isFindBookId = bookIdMatcher.find();
|
||||
}
|
||||
|
||||
totalPage = Integer.parseInt(totalPageMatcher.group(1));
|
||||
|
||||
}
|
||||
Pattern totalPagePatten = Pattern.compile(ruleBean.getTotalPagePatten());
|
||||
Matcher totalPageMatcher = totalPagePatten.matcher(bookListHtml);
|
||||
boolean isFindTotalPage = totalPageMatcher.find();
|
||||
if (isFindTotalPage) {
|
||||
|
||||
totalPage = Integer.parseInt(totalPageMatcher.group(1));
|
||||
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||
//捕获中断异常InterruptedException来退出线程。
|
||||
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
if (page == totalPage) {
|
||||
if (page >= totalPage) {
|
||||
// 第一遍采集完成,翻到第一页,继续第二次采集,适用于分页数比较少的最近更新列表
|
||||
page = 0;
|
||||
page = 1;
|
||||
try {
|
||||
// 第一遍采集完成,休眠1分钟
|
||||
Thread.sleep(Duration.ofMinutes(1));
|
||||
} catch (InterruptedException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
//1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时)
|
||||
//捕获中断异常InterruptedException来退出线程。
|
||||
//2.非阻塞过程中通过判断中断标志来退出线程。
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
page += 1;
|
||||
}
|
||||
|
||||
page += 1;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) {
|
||||
public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId)
|
||||
throws InterruptedException {
|
||||
|
||||
final AtomicBoolean parseResult = new AtomicBoolean(false);
|
||||
|
||||
@ -385,4 +410,5 @@ public class CrawlServiceImpl implements CrawlService {
|
||||
.render(RenderingStrategies.MYBATIS3);
|
||||
return crawlSourceMapper.selectMany(render);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -25,13 +25,9 @@ public class CrawlHttpClient {
|
||||
|
||||
private static final ThreadLocal<Integer> RETRY_COUNT = new ThreadLocal<>();
|
||||
|
||||
public String get(String url, String charset) {
|
||||
public String get(String url, String charset) throws InterruptedException {
|
||||
if (Objects.nonNull(intervalMin) && Objects.nonNull(intervalMax) && intervalMax > intervalMin) {
|
||||
try {
|
||||
Thread.sleep(random.nextInt(intervalMax - intervalMin + 1) + intervalMin);
|
||||
} catch (InterruptedException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
Thread.sleep(random.nextInt(intervalMax - intervalMin + 1) + intervalMin);
|
||||
}
|
||||
String body = HttpUtil.getByHttpClientWithChrome(url, charset);
|
||||
if (Objects.isNull(body) || body.length() < Constants.INVALID_HTML_LENGTH) {
|
||||
@ -41,7 +37,7 @@ public class CrawlHttpClient {
|
||||
return body;
|
||||
}
|
||||
|
||||
private String processErrorHttpResult(String url, String charset) {
|
||||
private String processErrorHttpResult(String url, String charset) throws InterruptedException{
|
||||
Integer count = RETRY_COUNT.get();
|
||||
if (count == null) {
|
||||
count = 0;
|
||||
|
@ -118,6 +118,9 @@
|
||||
示例:<b></p></b>
|
||||
<li><input type="text" id="descEnd" class="s_input icon_key" placeholder="小说简介结束截取字符串:">
|
||||
</li>
|
||||
示例:<b><span\s+class="allshow">([^/]+)</span></b>
|
||||
<li><textarea id="filterDesc"
|
||||
placeholder="过滤简介(多个内容换行)" rows="5" cols="52"></textarea></li>
|
||||
示例:<b>更新:(\d+-\d+-\d+\s\d+:\d+:\d+)</a></b>
|
||||
<li><input type="text" id="upadateTimePatten" class="s_input icon_key"
|
||||
placeholder="小说更新时间的正则表达式:"></li>
|
||||
@ -338,6 +341,9 @@
|
||||
|
||||
crawlRule.descEnd = descEnd;
|
||||
|
||||
var filterDesc = $("#filterDesc").val();
|
||||
crawlRule.filterDesc = filterDesc;
|
||||
|
||||
var upadateTimePatten = $("#upadateTimePatten").val();
|
||||
|
||||
if (upadateTimePatten.length > 0) {
|
||||
|
@ -119,6 +119,9 @@
|
||||
示例:<b></p></b>
|
||||
<li><input type="text" id="descEnd" class="s_input icon_key" placeholder="小说简介结束截取字符串:">
|
||||
</li>
|
||||
示例:<b><span\s+class="allshow">([^/]+)</span></b>
|
||||
<li><textarea id="filterDesc"
|
||||
placeholder="过滤简介(多个内容换行)" rows="5" cols="52"></textarea></li>
|
||||
示例:<b>更新:(\d+-\d+-\d+\s\d+:\d+:\d+)</a></b>
|
||||
<li><input type="text" id="upadateTimePatten" class="s_input icon_key"
|
||||
placeholder="小说更新时间的正则表达式:"></li>
|
||||
@ -266,6 +269,7 @@
|
||||
$("#visitCountPatten").val(crawlRule.visitCountPatten);
|
||||
$("#descStart").val(crawlRule.descStart);
|
||||
$("#descEnd").val(crawlRule.descEnd);
|
||||
$("#filterDesc").val(crawlRule.filterDesc);
|
||||
$("#upadateTimePatten").val(crawlRule.upadateTimePatten);
|
||||
$("#upadateTimeFormatPatten").val(crawlRule.upadateTimeFormatPatten);
|
||||
$("#bookIndexUrl").val(crawlRule.bookIndexUrl);
|
||||
@ -424,6 +428,9 @@
|
||||
|
||||
crawlRule.descEnd = descEnd;
|
||||
|
||||
var filterDesc = $("#filterDesc").val();
|
||||
crawlRule.filterDesc = filterDesc;
|
||||
|
||||
var upadateTimePatten = $("#upadateTimePatten").val();
|
||||
|
||||
if (upadateTimePatten.length > 0) {
|
||||
|
@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>novel</artifactId>
|
||||
<groupId>com.java2nb</groupId>
|
||||
<version>5.1.1</version>
|
||||
<version>5.1.5</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@ -61,6 +61,12 @@
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-openai-spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.lionsoul</groupId>
|
||||
<artifactId>ip2region</artifactId>
|
||||
<version>2.7.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
@ -2,12 +2,14 @@ package com.java2nb.novel.controller;
|
||||
|
||||
import com.java2nb.novel.core.bean.UserDetails;
|
||||
import com.java2nb.novel.core.enums.ResponseStatus;
|
||||
import com.java2nb.novel.core.utils.IpUtil;
|
||||
import com.java2nb.novel.entity.Book;
|
||||
import com.java2nb.novel.entity.BookCategory;
|
||||
import com.java2nb.novel.entity.BookComment;
|
||||
import com.java2nb.novel.entity.BookIndex;
|
||||
import com.java2nb.novel.service.BookContentService;
|
||||
import com.java2nb.novel.service.BookService;
|
||||
import com.java2nb.novel.service.IpLocationService;
|
||||
import com.java2nb.novel.vo.BookCommentVO;
|
||||
import com.java2nb.novel.vo.BookSettingVO;
|
||||
import com.java2nb.novel.vo.BookSpVO;
|
||||
@ -37,6 +39,8 @@ public class BookController extends BaseController {
|
||||
|
||||
private final Map<String, BookContentService> bookContentServiceMap;
|
||||
|
||||
private final IpLocationService ipLocationService;
|
||||
|
||||
/**
|
||||
* 查询首页小说设置列表数据
|
||||
*/
|
||||
@ -158,6 +162,7 @@ public class BookController extends BaseController {
|
||||
if (userDetails == null) {
|
||||
return RestResult.fail(ResponseStatus.NO_LOGIN);
|
||||
}
|
||||
comment.setLocation(ipLocationService.getLocation(IpUtil.getRealIp(request)));
|
||||
bookService.addBookComment(userDetails.getId(), comment);
|
||||
return RestResult.ok();
|
||||
}
|
||||
|
@ -0,0 +1,55 @@
|
||||
package com.java2nb.novel.core.config;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.lionsoul.ip2region.xdb.Searcher;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* IP 地址定位配置类
|
||||
*
|
||||
* @author xiongxiaoyang
|
||||
* @date 2025/6/30
|
||||
*/
|
||||
@Slf4j
|
||||
@Configuration
|
||||
public class IpLocationConfig {
|
||||
|
||||
/**
|
||||
* 使用 {@link Searcher} 实现高效的本地 IP 查询服务, 创建基于内存的 IP 地址查询对象,支持并发访问且仅需初始化一次。
|
||||
*
|
||||
* <p>该方法会将 ip2region.xdb 数据库文件加载到内存中,
|
||||
* 并构建一个线程安全的 {@link Searcher} 实例,可用于高效、并发的 IP 地址定位查询。</p>
|
||||
*
|
||||
* <p>{@link Searcher} 实例是线程安全的,可以作为全局单例在整个应用中跨线程使用。</p>
|
||||
*
|
||||
* <p>通过配置 destroyMethod="close",确保在 Spring 容器关闭时自动释放底层资源。</p>
|
||||
*/
|
||||
@Bean(destroyMethod = "close")
|
||||
public Searcher searcher() throws IOException {
|
||||
// 1、从 classpath 加载整个 xdb 到内存。
|
||||
try (InputStream inputStream = new ClassPathResource("ip2region.xdb").getInputStream()) {
|
||||
File tempDbFile = File.createTempFile("ip2region", ".xdb");
|
||||
try (FileOutputStream outputStream = new FileOutputStream(tempDbFile)) {
|
||||
byte[] buffer = new byte[1024];
|
||||
int bytesRead;
|
||||
while ((bytesRead = inputStream.read(buffer)) != -1) {
|
||||
outputStream.write(buffer, 0, bytesRead);
|
||||
}
|
||||
}
|
||||
// 确保程序退出时删除临时文件
|
||||
tempDbFile.deleteOnExit();
|
||||
byte[] cBuff = Searcher.loadContentFromFile(tempDbFile.getPath());
|
||||
|
||||
// 2、使用上述的 cBuff 创建一个完全基于内存的查询对象。
|
||||
return Searcher.newWithBuffer(cBuff);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,24 @@
|
||||
package com.java2nb.novel.service;
|
||||
|
||||
/**
|
||||
* IP 地址定位服务类
|
||||
*
|
||||
* <p>该服务用于实现 IP 地址到地理位置的查询功能,
|
||||
* 包括国家、省份、城市等信息。</p>
|
||||
*
|
||||
* <p>此类设计为 Spring 管理的 Service Bean,支持在 Controller 或其他 Service 中注入使用。</p>
|
||||
*
|
||||
* @author xiongxiaoyang
|
||||
* @date 2025/6/30
|
||||
*/
|
||||
public interface IpLocationService {
|
||||
|
||||
/**
|
||||
* 根据 IP 地址查询地理位置信息
|
||||
*
|
||||
* @param ip 待查询的 IP 地址(IPv4)
|
||||
* @return 如果是中国 IP,返回省份;否则返回国家
|
||||
*/
|
||||
String getLocation(String ip);
|
||||
|
||||
}
|
@ -0,0 +1,56 @@
|
||||
package com.java2nb.novel.service.impl;
|
||||
|
||||
import com.java2nb.novel.core.utils.IpUtil;
|
||||
import com.java2nb.novel.service.IpLocationService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.lionsoul.ip2region.xdb.Searcher;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* IpLocationService 实现类
|
||||
*
|
||||
* @author xiongxiaoyang
|
||||
* @date 2025/6/30
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class IpLocationServiceImpl implements IpLocationService {
|
||||
|
||||
private final Searcher searcher;
|
||||
|
||||
@Override
|
||||
public String getLocation(String ip) {
|
||||
try {
|
||||
// 示例返回:"中国|0|湖北省|武汉市|电信"
|
||||
String region = searcher.search(ip);
|
||||
String[] regions = region.split("\\|");
|
||||
if (regions.length > 0) {
|
||||
// 国家
|
||||
String country = regions[0];
|
||||
if ("0".equals(country)) {
|
||||
// 内网IP,直接获取本机公网IP
|
||||
String publicIp = IpUtil.getPublicIP();
|
||||
log.info("内网IP:{},本机公网IP:{}", ip, publicIp);
|
||||
if (StringUtils.hasText(publicIp)) {
|
||||
return getLocation(publicIp);
|
||||
}
|
||||
} else if ("中国".equals(country)) {
|
||||
// 是中国,则返回省份(第三个字段)
|
||||
String province = regions.length > 2 ? regions[2] : "未知地区";
|
||||
// 去掉最后一个“省”字
|
||||
return province.endsWith("省") ? province.substring(0, province.length() - 1) : province;
|
||||
} else {
|
||||
// 非中国,返回国家名
|
||||
return country;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
return "未知地区";
|
||||
}
|
||||
|
||||
}
|
BIN
novel-front/src/main/resources/ip2region.xdb
Normal file
BIN
novel-front/src/main/resources/ip2region.xdb
Normal file
Binary file not shown.
@ -4,7 +4,7 @@
|
||||
<mapper namespace="com.java2nb.novel.mapper.FrontBookCommentMapper">
|
||||
|
||||
<select id="listCommentByPage" resultType="com.java2nb.novel.vo.BookCommentVO">
|
||||
select t1.id,t1.book_id,t1.comment_content,t1.reply_count,t1.create_time,t2.username create_user_name,t2.user_photo create_user_photo
|
||||
select t1.id,t1.book_id,t1.comment_content,t1.location,t1.reply_count,t1.create_time,t2.username create_user_name,t2.user_photo create_user_photo
|
||||
from book_comment t1 inner join user t2 on t1.create_user_id = t2.id
|
||||
<trim>
|
||||
<if test="bookId != null">
|
||||
|
@ -135,7 +135,7 @@
|
||||
"<div class=\"user_heads fl\" vals=\"389\">" +
|
||||
"<img src=\""+(comment.createUserPhoto ? comment.createUserPhoto : '/images/man.png')+"\" class=\"user_head\" alt=\"\">" +
|
||||
"<span class=\"user_level1\" style=\"display: none;\">见习</span></div>" +
|
||||
"<ul class=\"pl_bar fr\">\t\t\t<li class=\"name\">"+(comment.createUserName.substr(0, 4) + "****" + comment.createUserName.substr(comment.createUserName.length - 3, 3))+"</li><li class=\"dec\">" +
|
||||
"<ul class=\"pl_bar fr\">\t\t\t<li class=\"name\">"+(comment.createUserName)+"<span style='padding-left: 10px' class=\"other\">"+(comment.location ? comment.location + "读者" : '')+"</span></li><li class=\"dec\">" +
|
||||
comment.commentContent+
|
||||
"</li><li class=\"other cf\">" +
|
||||
"<span class=\"time fl\">"+comment.createTime+"</span>" +
|
||||
|
@ -95,7 +95,7 @@
|
||||
<span class="block">暂无评论</span>
|
||||
</div>
|
||||
<div class="commentBar" id="commentPanel" th:style="${bookCommentPageBean.total == 0}? 'display:none'">
|
||||
<div th:each="comment: ${bookCommentPageBean.list}" class="comment_list cf"><div class="user_heads fl" vals="389"><img th:src="${comment.createUserPhoto}?${comment.createUserPhoto}:'/images/man.png'" class="user_head" alt=""><span class="user_level1" style="display: none;">见习</span></div><ul class="pl_bar fr"> <li class="name" th:text="${#strings.substring(comment.createUserName,0,4)}+'****'+${#strings.substring(comment.createUserName,#strings.length(comment.createUserName)-3,#strings.length(comment.createUserName))}"></li><li class="dec" th:utext="${comment.commentContent}"></li><li class="other cf"><span class="time fl" th:text="${#calendars.format(comment.createTime, 'yyyy-MM-dd HH:mm:ss')}"></span><span class="fr"><a href="javascript:void(0);" onclick="javascript:BookDetail.AddAgreeTotal(77,this);" class="zan" style="display: none;">赞<i class="num">(0)</i></a></span></li> </ul> </div>
|
||||
<div th:each="comment: ${bookCommentPageBean.list}" class="comment_list cf"><div class="user_heads fl" vals="389"><img th:src="${comment.createUserPhoto}?${comment.createUserPhoto}:'/images/man.png'" class="user_head" alt=""><span class="user_level1" style="display: none;">见习</span></div><ul class="pl_bar fr"> <li class="name"><span th:text="${#strings.substring(comment.createUserName,0,4)}+'****'+${#strings.substring(comment.createUserName,#strings.length(comment.createUserName)-3,#strings.length(comment.createUserName))}"></span><span style="padding-left: 10px" class="other" th:if="${comment.location}" th:text="${comment.location} + '读者'"></span></span></li><li class="dec" th:utext="${comment.commentContent}"></li><li class="other cf"><span class="time fl" th:text="${#calendars.format(comment.createTime, 'yyyy-MM-dd HH:mm:ss')}"></span><span class="fr"><a href="javascript:void(0);" onclick="javascript:BookDetail.AddAgreeTotal(77,this);" class="zan" style="display: none;">赞<i class="num">(0)</i></a></span></li> </ul> </div>
|
||||
|
||||
</div>
|
||||
|
||||
@ -319,7 +319,7 @@
|
||||
"<div class=\"user_heads fl\" vals=\"389\">" +
|
||||
"<img src=\""+(comment.createUserPhoto ? comment.createUserPhoto : '/images/man.png')+"\" class=\"user_head\" alt=\"\">" +
|
||||
"<span class=\"user_level1\" style=\"display: none;\">见习</span></div>" +
|
||||
"<ul class=\"pl_bar fr\">\t\t\t<li class=\"name\">"+(comment.createUserName)+"</li><li class=\"dec\">" +
|
||||
"<ul class=\"pl_bar fr\">\t\t\t<li class=\"name\">"+(comment.createUserName)+"<span style='padding-left: 10px' class=\"other\">"+(comment.location ? comment.location + "读者" : '')+"</span></li><li class=\"dec\">" +
|
||||
comment.commentContent+
|
||||
"</li><li class=\"other cf\">" +
|
||||
"<span class=\"time fl\">"+comment.createTime+"</span>" +
|
||||
|
Reference in New Issue
Block a user