diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..229bd24 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,79 @@ +name: Create novel-plus Maven Release with ZIPs + +on: + push: + # 匹配所有以'v'开头的标签 + tags: + - 'v*' + +jobs: + build-and-release: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up JDK 21 + uses: actions/setup-java@v3 + with: + java-version: '21' + # 可选,默认是 temurin,也可以选择其他发行版 + distribution: 'temurin' + + - name: Build project with Maven + run: mvn clean install -DskipTests=true -Pcentral-repo + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # 使用 tag_name 而不是 github.ref + tag_name: ${{ github.ref_name }} + release_name: novel-plus ${{ github.ref_name }} + draft: false + prerelease: false + + + # 使用 action 来替代直接 curl 进行上传 + - name: Upload sql.zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ github.workspace }}/novel-common/target/build/sql.zip + asset_name: sql.zip + asset_content_type: application/zip + + - name: Upload novel-crawl.zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ github.workspace }}/novel-crawl/target/build/novel-crawl.zip + asset_name: novel-crawl.zip + asset_content_type: application/zip + + - name: Upload novel-front.zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ github.workspace }}/novel-front/target/build/novel-front.zip + asset_name: novel-front.zip + asset_content_type: application/zip + + - name: Upload novel-admin.zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ github.workspace }}/novel-admin/target/build/novel-admin.zip + asset_name: novel-admin.zip + asset_content_type: application/zip \ No newline at end of file diff --git a/README.md b/README.md index baffe3c..7b9ea06 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,28 @@ -[![index]( https://youdoc.github.io/img/tencent.jpg )]( https://cloud.tencent.com/act/cps/redirect?redirect=2446&cps_key=736e609d66e0ac4e57813316cec6fd0b&from=console ) - +

+ AD +

Github stars Github forks Gitee stars Gitee forks - visitors

- 👉 官网 | 👉 项目演示 | 👉 安装教程 + 👉 官网 | 👉 演示站点 | 👉 安装教程

## 项目介绍 novel-plus 是一个多端(PC、WAP)阅读,功能完善的原创文学 CMS 系统。由前台门户系统、作家后台管理系统、平台后台管理系统和爬虫管理系统等多个子系统构成,包括小说推荐、作品检索、小说排行、小说阅读、小说评论、会员中心、作家专区等功能,支持自定义多模版、可拓展的多种小说内容存储方式(内置数据库分表存储和 -TXT 文本存储)、阅读主题切换、多爬虫源自动采集和更新数据、会员充值、订阅模式、新闻发布和实时统计报表。 +TXT 文本存储)、阅读主题切换、多爬虫源自动采集和更新数据、AI写作、会员充值、订阅模式、新闻发布和实时统计报表。 ## 项目地址 - 学习版:[GitHub](https://github.com/201206030/novel) | [码云](https://gitee.com/novel_dev_team/novel) - | [保姆级教程](https://docs.xxyopen.com) -- **应用版**:[GitHub](https://github.com/201206030/novel-plus) | [码云](https://gitee.com/novel_dev_team/novel-plus) + | [保姆级教程](https://docs.xxyopen.com) +- **应用版**:[GitHub](https://github.com/201206030/novel-plus) | [码云](https://gitee.com/novel_dev_team/novel-plus) | [演示站点](http://117.72.165.13:8888) - 微服务版:[GitHub](https://github.com/201206030/novel-cloud) | [码云](https://gitee.com/novel_dev_team/novel-cloud) ## 项目结构 @@ -38,24 +38,25 @@ novel-plus -- 父工程 ## 技术选型 -| 技术 | 说明 -|---------------------| --------------------------- -| Spring Boot | Spring 应用快速开发脚手架 -| MyBatis | 持久层 ORM 框架 -| MyBatis Dynamic SQL | Mybatis 动态 sql -| PageHelper | MyBatis 分页插件 -| MyBatis Generator | 持久层代码生成插件 -| Sharding-JDBC | 代码层分库分表中间件 -| JJWT | JWT 登录支持 -| Spring Security | 安全框架 -| Apache Shiro | 安全框架 -| Redis | 缓存方案 -| Aliyun OSS | 阿里云对象存储服务(图片存储备选方案) -| Lombok | 简化对象封装工具 -| Docker | 应用容器引擎 -| MySQL | 数据库服务 -| Thymeleaf | 模板引擎 -| Layui | 前端 UI 框架 +| 技术 | 说明 +|---------------------|--------------------- +| Spring Boot | Spring 应用快速开发脚手架 +| Spring AI | Spring 官方 AI 框架 +| MyBatis | 持久层 ORM 框架 +| MyBatis Dynamic SQL | Mybatis 动态 sql +| PageHelper | MyBatis 分页插件 +| MyBatis Generator | 持久层代码生成插件 +| Sharding-JDBC | 代码层分库分表中间件 +| JJWT | JWT 登录支持 +| Spring Security | 安全框架 +| Apache Shiro | 安全框架 +| Redis | 缓存方案 +| Aliyun OSS | 阿里云对象存储服务(图片存储备选方案) +| Lombok | 简化对象封装工具 +| Docker | 应用容器引擎 +| MySQL | 数据库服务 +| Thymeleaf | 模板引擎 +| Layui | 前端 UI 框架 ## 项目截图 @@ -69,6 +70,39 @@ novel-plus -- 父工程 https://www.bilibili.com/video/BV18e41197xs +## AI 功能 + +novel-plus 5.x 已集成 Spring 官方最新发布的 Spring AI 框架,并推出多项 AI 功能: + +1. v5.0.0 版本在小说章节发布页面的文本编辑器中集成了多项智能编辑功能,包括 AI 扩写、缩写、续写及文本润色等。这些功能的设计灵感来源于百家号文章编辑器中的 AI 助手。 +2. v5.1.0 版本在小说发布页面,新增 AI 生成封面图功能。若作家未上传自定义封面图,系统将根据小说信息自动生成封面图。 + +目前,AI 功能仍处于实验阶段,仅实现了基础的核心功能。我们非常重视用户的实际使用体验和反馈,未来将根据用户需求和使用情况,持续优化和调整该功能。如果用户反馈积极,我们计划进一步开发更高级的 AI 功能,例如自动生成有声小说、智能情节推荐等,以全面提升 novel-plus 的创作能力和用户体验。 + +我们将持续关注 AI 技术的发展,并致力于将其与小说创作场景深度融合,为用户带来更智能、更便捷的创作工具。 + +novel-plus 项目默认使用的是第三方大模型服务平台[硅基流动](https://cloud.siliconflow.cn/i/DOgMRH9S)提供的 API(兼容 OpenAI 的相关接口,可直接通过 Spring AI 框架调用),采用的 AI 模型有对话模型`deepseek-ai/DeepSeek-R1-Distill-Llama-8B`(DeepSeek-R1 的蒸馏版本,免费使用)和生图模型`Kwai-Kolors/Kolors`(快手 Kolors 团队开发的文本到图像生成模型,免费使用)。只需注册一个硅基流动账号,创建一个 API 密钥,并将其添加到 novel-plus 项目 novel-front 模块的 yaml 配置文件中,即可体验 novel-plus 项目的 AI 写作功能。 + +```yaml +spring: + ai: + openai: + image: + enabled: true + base-url: https://api.siliconflow.cn + api-key: sk-rrrupturhdofbiqzjutduuiceecpvfqlnvmgcyiaipbdikoi + options: + model: Kwai-Kolors/Kolors + response_format: URL + api-key: sk-rrrupturhdofbiqzjutduuiceecpvfqlnvmgcyiaipbdikoi + base-url: https://api.siliconflow.cn + chat: + options: + model: deepseek-ai/DeepSeek-R1-Distill-Llama-8B +``` + +⚠️ novel-plus 项目默认使用的都是免费 AI 模型,生成效果有限。如果对生成内容有更高的要求,建议选用付费的 AI 模型。 + ## 增值服务 👉 [了解详情](https://novel.xxyopen.com/service.htm) @@ -93,3 +127,5 @@ https://www.bilibili.com/video/BV18e41197xs ## 免责声明 本项目提供的爬虫工具仅用于采集项目初期的测试数据,请勿用于商业盈利。 用户使用本系统从事任何违法违规的事情,一切后果由用户自行承担,作者不承担任何责任。 + + diff --git a/config/shardingsphere-jdbc.yml b/config/shardingsphere-jdbc.yml new file mode 100644 index 0000000..057cc7e --- /dev/null +++ b/config/shardingsphere-jdbc.yml @@ -0,0 +1,53 @@ +mode: + # 单机模式 + type: Standalone + # 元数据持久化 + repository: + # 数据库持久化 + type: JDBC + +# 数据源配置 +dataSources: + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.mysql.cj.jdbc.Driver + jdbcUrl: jdbc:mysql://localhost:3306/novel_plus?allowPublicKeyRetrieval=true&useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai + username: root + password: test123456 + ds_2: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://localhost:3306/information_schema?allowPublicKeyRetrieval=true&useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai + username: root + password: test123456 +# 规则配置 +rules: + - !SINGLE + tables: + - "*.*" + - !SHARDING + tables: # 数据分片规则配置 + book_content: + # 分库策略,缺省表示使用默认分库策略 + actualDataNodes: ds_${1}.book_content${0..9} + # 分表策略 + tableStrategy: + standard: + # 分片列名称 + shardingColumn: index_id + # 分片算法名称 + shardingAlgorithmName: bookContentSharding + + shardingAlgorithms: + bookContentSharding: + # 行表达式分片算法,使用 Groovy 的表达式,提供对 SQL 语句中的 = 和 IN 的分片操作支持 + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: book_content${index_id % 10} + + + +props: + # 是否在日志中打印 SQL + sql-show: true diff --git a/doc/sql/20250317.sql b/doc/sql/20250317.sql new file mode 100644 index 0000000..56f1808 --- /dev/null +++ b/doc/sql/20250317.sql @@ -0,0 +1,44 @@ +INSERT INTO crawl_source (source_name, crawl_rule, source_status, create_time, update_time) +VALUES ('香书小说网', '{ + "bookListUrl": "http://www.xbiqugu.la/fenlei/{catId}_{page}.html", + "catIdRule": { + "catId1": "1", + "catId2": "2", + "catId3": "3", + "catId4": "4", + "catId5": "6", + "catId6": "5" + }, + "bookIdPatten": "", + "pagePatten": "(\\\\d+)/\\\\d+", + "totalPagePatten": "\\\\d+/(\\\\d+)", + "bookDetailUrl": "http://www.xbiqugu.la/{bookId}/", + "bookNamePatten": "

([^/]+)

", + "authorNamePatten": "者:([^/]+)

", + "picUrlPatten": "src=\\"(http://www.xbiqugu.la/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\"", + "bookStatusRule": {}, + "descStart": "
", + "descEnd": "
", + "upadateTimePatten": "

最后更新:(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)

", + "upadateTimeFormatPatten": "yyyy-MM-dd HH:mm:ss", + "bookIndexUrl": "http://www.xbiqugu.la/{bookId}/", + "indexIdPatten": "[^/]+", + "indexNamePatten": "([^/]+)", + "bookContentUrl": "http://www.xbiqugu.la/{bookId}/{indexId}.html", + "contentStart": "
", + "contentEnd": "

", + "filterContent":"\\\\s*([^/]+)\\\\s*

" +}', 0, '2024-06-01 10:11:39', '2024-06-01 10:11:39'); + + +update crawl_source +set crawl_rule = replace(crawl_rule, 'ibiquzw.org', 'biquxs.info') +where id = 16; + +delete +from sys_menu +where menu_id = 104; + +delete +from sys_menu +where menu_id = 57; \ No newline at end of file diff --git a/doc/sql/20250630.sql b/doc/sql/20250630.sql new file mode 100644 index 0000000..23ef4c6 --- /dev/null +++ b/doc/sql/20250630.sql @@ -0,0 +1,3 @@ +alter table book_comment add column location varchar(50) DEFAULT NULL COMMENT '地理位置' after comment_content ; + + diff --git a/doc/sql/20250711.sql b/doc/sql/20250711.sql new file mode 100644 index 0000000..e50a849 --- /dev/null +++ b/doc/sql/20250711.sql @@ -0,0 +1,3 @@ +alter table crawl_single_task add column crawl_chapters int DEFAULT 0 COMMENT '采集章节数量' after exc_count ; + + diff --git a/doc/sql/20250712.sql b/doc/sql/20250712.sql new file mode 100644 index 0000000..d2eb17a --- /dev/null +++ b/doc/sql/20250712.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS `book_comment_reply`; +CREATE TABLE `book_comment_reply` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `comment_id` bigint(20) DEFAULT NULL COMMENT '评论ID', + `reply_content` varchar(512) DEFAULT NULL COMMENT '回复内容', + `location` varchar(50) DEFAULT NULL COMMENT '地理位置', + `audit_status` tinyint(1) DEFAULT '0' COMMENT '审核状态,0:待审核,1:审核通过,2:审核不通过', + `create_time` datetime DEFAULT NULL COMMENT '回复用户ID', + `create_user_id` bigint(20) DEFAULT NULL COMMENT '回复时间', + PRIMARY KEY (`id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 COMMENT ='小说评论回复表'; \ No newline at end of file diff --git a/doc/sql/novel_plus.sql b/doc/sql/novel_plus.sql index dd7b73b..577d3fd 100644 --- a/doc/sql/novel_plus.sql +++ b/doc/sql/novel_plus.sql @@ -1,7 +1,5 @@ -CREATE - database if NOT EXISTS `novel_plus` default character set utf8mb4 collate utf8mb4_unicode_ci; -use - `novel_plus`; +CREATE database if NOT EXISTS `novel_plus` default character set utf8mb4 collate utf8mb4_unicode_ci; +use `novel_plus`; SET NAMES utf8mb4; @@ -3105,4 +3103,75 @@ where id = 16; update website_info set logo = '/images/logo.png', logo_dark='/images/logo.png' -where id = 1; \ No newline at end of file +where id = 1; + + +INSERT INTO crawl_source (source_name, crawl_rule, source_status, create_time, update_time) +VALUES ('香书小说网', '{ + "bookListUrl": "http://www.xbiqugu.net/fenlei/{catId}_{page}.html", + "catIdRule": { + "catId1": "1", + "catId2": "2", + "catId3": "3", + "catId4": "4", + "catId5": "6", + "catId6": "5" + }, + "bookIdPatten": "", + "pagePatten": "(\\\\d+)/\\\\d+", + "totalPagePatten": "\\\\d+/(\\\\d+)", + "bookDetailUrl": "http://www.xbiqugu.net/{bookId}/", + "bookNamePatten": "

([^/]+)

", + "authorNamePatten": "者:([^/]+)

", + "picUrlPatten": "src=\\"(http://www.xbiqugu.net/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\"", + "bookStatusRule": {}, + "descStart": "
", + "descEnd": "
", + "upadateTimePatten": "

最后更新:(\\\\d+-\\\\d+-\\\\d+\\\\s\\\\d+:\\\\d+:\\\\d+)

", + "upadateTimeFormatPatten": "yyyy-MM-dd HH:mm:ss", + "bookIndexUrl": "http://www.xbiqugu.net/{bookId}/", + "indexIdPatten": "[^/]+", + "indexNamePatten": "([^/]+)", + "bookContentUrl": "http://www.xbiqugu.net/{bookId}/{indexId}.html", + "contentStart": "
", + "contentEnd": "

", + "filterContent":"\\\\s*([^/]+)\\\\s*

" +}', 0, '2024-06-01 10:11:39', '2024-06-01 10:11:39'); + + +update crawl_source +set crawl_rule = replace(crawl_rule, 'ibiquzw.org', 'biquxs.info') +where id = 16; + + +update crawl_source +set crawl_rule = replace(crawl_rule, 'xbiqugu.net', 'xbiqugu.la'); + +delete +from sys_menu +where menu_id = 104; + +delete +from sys_menu +where menu_id = 57; + + +alter table book_comment add column location varchar(50) DEFAULT NULL COMMENT '地理位置' after comment_content ; + + +alter table crawl_single_task add column crawl_chapters int DEFAULT 0 COMMENT '采集章节数量' after exc_count ; + + +DROP TABLE IF EXISTS `book_comment_reply`; +CREATE TABLE `book_comment_reply` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `comment_id` bigint(20) DEFAULT NULL COMMENT '评论ID', + `reply_content` varchar(512) DEFAULT NULL COMMENT '回复内容', + `location` varchar(50) DEFAULT NULL COMMENT '地理位置', + `audit_status` tinyint(1) DEFAULT '0' COMMENT '审核状态,0:待审核,1:审核通过,2:审核不通过', + `create_time` datetime DEFAULT NULL COMMENT '回复用户ID', + `create_user_id` bigint(20) DEFAULT NULL COMMENT '回复时间', + PRIMARY KEY (`id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 COMMENT ='小说评论回复表'; \ No newline at end of file diff --git a/novel-admin/pom.xml b/novel-admin/pom.xml index bbf634f..608bb84 100644 --- a/novel-admin/pom.xml +++ b/novel-admin/pom.xml @@ -5,7 +5,7 @@ com.java2nb novel-admin - 4.3.0 + 5.2.0 jar novel-admin @@ -14,18 +14,14 @@ org.springframework.boot spring-boot-starter-parent - 2.1.18.RELEASE + 2.7.18 - UTF-8 - UTF-8 - 1.8 + 21 1.7 - 5.22.0 - 3.0.0 - 2.15.1 + 5.5.1 @@ -60,35 +56,24 @@ net.sourceforge.nekohtml nekohtml + + + org.hibernate.validator + hibernate-validator + mysql mysql-connector-java 8.0.29 - - org.mybatis - mybatis - 3.5.6 - org.mybatis.spring.boot mybatis-spring-boot-starter 1.1.1 - - - com.alibaba - druid - 1.2.9 - - - org.apache.commons - commons-lang3 - 3.6 - commons-configuration commons-configuration @@ -138,6 +123,12 @@ org.apache.velocity velocity 1.7 + + + commons-lang + commons-lang + + @@ -166,6 +157,12 @@ io.springfox springfox-swagger2 2.6.1 + + + guava + com.google.guava + + io.springfox @@ -195,24 +192,29 @@ org.apache.commons commons-text 1.4 + + + commons-lang3 + org.apache.commons + + + - io.shardingsphere - sharding-jdbc-spring-boot-starter - ${sharding.jdbc.version} + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere-jdbc.version} - - io.shardingsphere - sharding-jdbc-spring-namespace - ${sharding.jdbc.version} + org.yaml + snakeyaml + 2.2 - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} + com.h2database + h2 + runtime @@ -330,4 +332,35 @@ + + + + + central-repo + + + central + https://repo.maven.apache.org/maven2/ + + true + + + false + + + + + + central-plugin + https://repo.maven.apache.org/maven2/ + + true + + + false + + + + + diff --git a/novel-admin/src/main/build/config/application-prod.yml b/novel-admin/src/main/build/config/application-prod.yml deleted file mode 100644 index 73d0c2f..0000000 --- a/novel-admin/src/main/build/config/application-prod.yml +++ /dev/null @@ -1,20 +0,0 @@ -#端口号 -server: - port: 8088 -spring: - datasource: - url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - redis: - host: 127.0.0.1 - port: 6379 - password: test123456 - -sharding: - jdbc: - datasource: - ds0: - jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 diff --git a/novel-admin/src/main/build/config/application.yml b/novel-admin/src/main/build/config/application.yml new file mode 100644 index 0000000..597dfad --- /dev/null +++ b/novel-admin/src/main/build/config/application.yml @@ -0,0 +1,9 @@ +#端口号 +server: + port: 8088 +spring: + redis: + host: 127.0.0.1 + port: 6379 + password: test123456 + diff --git a/novel-admin/src/main/build/config/shardingsphere-jdbc.yml b/novel-admin/src/main/build/config/shardingsphere-jdbc.yml new file mode 100644 index 0000000..8659c28 --- /dev/null +++ b/novel-admin/src/main/build/config/shardingsphere-jdbc.yml @@ -0,0 +1,47 @@ +mode: + # 单机模式 + type: Standalone + # 元数据持久化 + repository: + # 数据库持久化 + type: JDBC + +# 数据源配置 +dataSources: + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.mysql.cj.jdbc.Driver + jdbcUrl: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai + username: root + password: test123456 +# 规则配置 +rules: + - !SINGLE + tables: + - "*.*" + - !SHARDING + tables: # 数据分片规则配置 + book_content: + # 分库策略,缺省表示使用默认分库策略 + actualDataNodes: ds_${1}.book_content${0..9} + # 分表策略 + tableStrategy: + standard: + # 分片列名称 + shardingColumn: index_id + # 分片算法名称 + shardingAlgorithmName: bookContentSharding + + shardingAlgorithms: + bookContentSharding: + # 行表达式分片算法,使用 Groovy 的表达式,提供对 SQL 语句中的 = 和 IN 的分片操作支持 + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: book_content${index_id % 10} + + + +props: + # 是否在日志中打印 SQL + sql-show: true diff --git a/novel-admin/src/main/java/com/java2nb/AdminApplication.java b/novel-admin/src/main/java/com/java2nb/AdminApplication.java index cb20c22..29b7a12 100644 --- a/novel-admin/src/main/java/com/java2nb/AdminApplication.java +++ b/novel-admin/src/main/java/com/java2nb/AdminApplication.java @@ -11,27 +11,37 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.transaction.annotation.EnableTransactionManagement; +import javax.sql.DataSource; import java.net.InetAddress; +import java.sql.Connection; @EnableTransactionManagement @ServletComponentScan @MapperScan("com.java2nb.*.dao") @SpringBootApplication(exclude = { - org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class + org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class }) @EnableCaching @Slf4j public class AdminApplication { + public static void main(String[] args) { SpringApplication.run(AdminApplication.class, args); } @Bean - public CommandLineRunner commandLineRunner(ApplicationContext ctx) { + public CommandLineRunner commandLineRunner(ApplicationContext ctx, DataSource dataSource) { return args -> { - log.info("项目启动啦,访问路径:{}", "http://" + InetAddress.getLocalHost().getHostAddress() + ":" + ctx.getEnvironment().getProperty("server.port")); + log.info("创建连接池..."); + try (Connection connection = dataSource.getConnection()) { + log.info("连接池已创建."); + log.info("数据库:{}", connection.getMetaData().getDatabaseProductName()); + log.info("数据库版本:{}", connection.getMetaData().getDatabaseProductVersion()); + } + log.info("项目启动啦,访问路径:{}", + "http://" + InetAddress.getLocalHost().getHostAddress() + ":" + ctx.getEnvironment() + .getProperty("server.port")); }; } - } diff --git a/novel-admin/src/main/java/com/java2nb/common/aspect/WebLogAspect.java b/novel-admin/src/main/java/com/java2nb/common/aspect/WebLogAspect.java index 889fdb6..0d6dce5 100644 --- a/novel-admin/src/main/java/com/java2nb/common/aspect/WebLogAspect.java +++ b/novel-admin/src/main/java/com/java2nb/common/aspect/WebLogAspect.java @@ -1,19 +1,16 @@ package com.java2nb.common.aspect; -import com.java2nb.common.utils.HttpContextUtils; +import com.java2nb.common.utils.IPUtils; import org.aspectj.lang.JoinPoint; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.*; -import org.aspectj.lang.reflect.MethodSignature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; -import sun.net.util.IPAddressUtil; import javax.servlet.http.HttpServletRequest; -import java.lang.reflect.Method; import java.util.Arrays; @Aspect @@ -37,11 +34,10 @@ public class WebLogAspect { logger.info("请求地址 : " + request.getRequestURL().toString()); logger.info("HTTP METHOD : " + request.getMethod()); // 获取真实的ip地址 - //logger.info("IP : " + IPAddressUtil.getClientIpAddress(request)); + logger.info("IP : " + IPUtils.getIpAddr(request)); logger.info("CLASS_METHOD : " + joinPoint.getSignature().getDeclaringTypeName() + "." - + joinPoint.getSignature().getName()); + + joinPoint.getSignature().getName()); logger.info("参数 : " + Arrays.toString(joinPoint.getArgs())); -// loggger.info("参数 : " + joinPoint.getArgs()); } diff --git a/novel-admin/src/main/java/com/java2nb/common/config/DruidDBConfig.java b/novel-admin/src/main/java/com/java2nb/common/config/DruidDBConfig.java deleted file mode 100644 index 92a2383..0000000 --- a/novel-admin/src/main/java/com/java2nb/common/config/DruidDBConfig.java +++ /dev/null @@ -1,132 +0,0 @@ -package com.java2nb.common.config; -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.support.http.StatViewServlet; -import com.alibaba.druid.support.http.WebStatFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.web.servlet.FilterRegistrationBean; -import org.springframework.boot.web.servlet.ServletRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; - -import javax.sql.DataSource; -import java.sql.SQLException; - -/** - * Created by PrimaryKey on 17/2/4. - */ -@SuppressWarnings("AlibabaRemoveCommentedCode") -@Configuration -public class DruidDBConfig { - private Logger logger = LoggerFactory.getLogger(DruidDBConfig.class); - @Value("${spring.datasource.url}") - private String dbUrl; - - @Value("${spring.datasource.username}") - private String username; - - @Value("${spring.datasource.password}") - private String password; - - @Value("${spring.datasource.driverClassName}") - private String driverClassName; - - @Value("${spring.datasource.initialSize}") - private int initialSize; - - @Value("${spring.datasource.minIdle}") - private int minIdle; - - @Value("${spring.datasource.maxActive}") - private int maxActive; - - @Value("${spring.datasource.maxWait}") - private int maxWait; - - @Value("${spring.datasource.timeBetweenEvictionRunsMillis}") - private int timeBetweenEvictionRunsMillis; - - @Value("${spring.datasource.minEvictableIdleTimeMillis}") - private int minEvictableIdleTimeMillis; - - @Value("${spring.datasource.validationQuery}") - private String validationQuery; - - @Value("${spring.datasource.testWhileIdle}") - private boolean testWhileIdle; - - @Value("${spring.datasource.testOnBorrow}") - private boolean testOnBorrow; - - @Value("${spring.datasource.testOnReturn}") - private boolean testOnReturn; - - @Value("${spring.datasource.poolPreparedStatements}") - private boolean poolPreparedStatements; - - @Value("${spring.datasource.maxPoolPreparedStatementPerConnectionSize}") - private int maxPoolPreparedStatementPerConnectionSize; - - @Value("${spring.datasource.filters}") - private String filters; - - @Value("{spring.datasource.connectionProperties}") - private String connectionProperties; - - @Bean(initMethod = "init", destroyMethod = "close") //声明其为Bean实例 - @Primary //在同样的DataSource中,首先使用被标注的DataSource - public DataSource dataSource() { - DruidDataSource datasource = new DruidDataSource(); - - datasource.setUrl(this.dbUrl); - datasource.setUsername(username); - datasource.setPassword(password); - datasource.setDriverClassName(driverClassName); - - //configuration - datasource.setInitialSize(initialSize); - datasource.setMinIdle(minIdle); - datasource.setMaxActive(maxActive); - datasource.setMaxWait(maxWait); - datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); - datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); - datasource.setValidationQuery(validationQuery); - datasource.setTestWhileIdle(testWhileIdle); - datasource.setTestOnBorrow(testOnBorrow); - datasource.setTestOnReturn(testOnReturn); - datasource.setPoolPreparedStatements(poolPreparedStatements); - datasource.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize); - try { - datasource.setFilters(filters); - } catch (SQLException e) { - logger.error("druid configuration initialization filter", e); - } - datasource.setConnectionProperties(connectionProperties); - - return datasource; - } - - @Bean - public ServletRegistrationBean druidServlet() { - ServletRegistrationBean reg = new ServletRegistrationBean(); - reg.setServlet(new StatViewServlet()); - reg.addUrlMappings("/druid/*"); - reg.addInitParameter("allow", ""); //白名单 - return reg; - } - - @Bean public FilterRegistrationBean filterRegistrationBean() { - FilterRegistrationBean filterRegistrationBean = new FilterRegistrationBean(); - filterRegistrationBean.setFilter(new WebStatFilter()); - filterRegistrationBean.addUrlPatterns("/*"); - filterRegistrationBean.addInitParameter("exclusions", "*.js,*.gif,*.jpg,*.png,*.css,*.ico,/druid/*"); - filterRegistrationBean.addInitParameter("profileEnable", "true"); - filterRegistrationBean.addInitParameter("principalCookieName","USER_COOKIE"); - filterRegistrationBean.addInitParameter("principalSessionName","USER_SESSION"); - filterRegistrationBean.addInitParameter("DruidWebStatFilter","/*"); - return filterRegistrationBean; - } -} - diff --git a/novel-admin/src/main/java/com/java2nb/common/config/Swagger2Config.java b/novel-admin/src/main/java/com/java2nb/common/config/Swagger2Config.java deleted file mode 100644 index 90b912c..0000000 --- a/novel-admin/src/main/java/com/java2nb/common/config/Swagger2Config.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.java2nb.common.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import springfox.documentation.builders.ApiInfoBuilder; -import springfox.documentation.builders.PathSelectors; -import springfox.documentation.builders.RequestHandlerSelectors; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.service.Contact; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; - -/** - * ${DESCRIPTION} - * - * @author xiongxy - * @create 2019-11-02 23:53 - */ -@EnableSwagger2 -@Configuration -public class Swagger2Config { - - @Bean - public Docket createRestApi() { - return new Docket(DocumentationType.SWAGGER_2) - .apiInfo(apiInfo()) - .select() - //为当前包路径 - .apis(RequestHandlerSelectors.any()) - .paths(PathSelectors.any()) - .build(); - } - - //构建 api文档的详细信息函数 - private ApiInfo apiInfo() { - return new ApiInfoBuilder() - //页面标题 - .title("功能测试") - //创建人 - .contact(new Contact("xiongxy", "1179705413@qq.com", "1179705413@qq.com")) - //版本号 - .version("1.0") - //描述 - .description("API 描述") - .build(); - } -} \ No newline at end of file diff --git a/novel-admin/src/main/java/com/java2nb/common/dao/GeneratorMapper.java b/novel-admin/src/main/java/com/java2nb/common/dao/GeneratorMapper.java index 38996bb..914affc 100644 --- a/novel-admin/src/main/java/com/java2nb/common/dao/GeneratorMapper.java +++ b/novel-admin/src/main/java/com/java2nb/common/dao/GeneratorMapper.java @@ -9,25 +9,25 @@ import java.util.Map; public interface GeneratorMapper { @Select( - "select table_name tableName, engine, table_comment tableComment, create_time createTime from information_schema.tables" + "select table_name tableName, engine, table_comment tableComment, create_time createTime from tables" + " where table_schema = 'novel_plus' and table_name like concat('%',#{tableName},'%')") List> list(@Param("tableName") String tableName); - @Select("select count(*) from information_schema.tables where table_schema = 'novel_plus'") + @Select("select count(*) from tables where table_schema = 'novel_plus'") int count(Map map); @Select( - "select table_name tableName, engine, table_comment tableComment, create_time createTime from information_schema.tables \r\n" + "select table_name tableName, engine, table_comment tableComment, create_time createTime from tables \r\n" + " where table_schema = 'novel_plus' and table_name = #{tableName}") Map get(String tableName); @Select( - "select column_name columnName, data_type dataType, column_comment columnComment, column_key columnKey, extra from information_schema.columns\r\n" + "select column_name columnName, data_type dataType, column_comment columnComment, column_key columnKey, extra from columns\r\n" + " where table_name = #{tableName} and table_schema = 'novel_plus' order by ordinal_position") List> listColumns(String tableName); @Select( - "select column_name columnName, data_type dataType, column_comment columnComment, column_key columnKey, extra from information_schema.columns\r\n" + "select column_name columnName, data_type dataType, column_comment columnComment, column_key columnKey, extra from columns\r\n" + " where table_name = #{tableName} and table_schema = 'novel_plus' and column_key = 'PRI' limit 1") Map getPriColumn(String tableName); } diff --git a/novel-admin/src/main/java/com/java2nb/common/exception/MainsiteErrorController.java b/novel-admin/src/main/java/com/java2nb/common/exception/MainsiteErrorController.java index 46da1fd..1a9fc10 100644 --- a/novel-admin/src/main/java/com/java2nb/common/exception/MainsiteErrorController.java +++ b/novel-admin/src/main/java/com/java2nb/common/exception/MainsiteErrorController.java @@ -2,32 +2,27 @@ package com.java2nb.common.exception; import com.java2nb.common.utils.R; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.web.servlet.error.ErrorAttributes; import org.springframework.boot.web.servlet.error.ErrorController; -import org.springframework.http.HttpStatus; -import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.servlet.ModelAndView; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.util.Map; @RestController public class MainsiteErrorController implements ErrorController { - private Logger logger = LoggerFactory.getLogger(getClass()); + private static final String ERROR_PATH = "/error"; @Autowired ErrorAttributes errorAttributes; @RequestMapping( - value = {ERROR_PATH}, - produces = {"text/html"} + value = {ERROR_PATH}, + produces = {"text/html"} ) public ModelAndView errorHtml(HttpServletRequest request, HttpServletResponse response) { int code = response.getStatus(); @@ -58,9 +53,4 @@ public class MainsiteErrorController implements ErrorController { } } - @Override - public String getErrorPath() { - // TODO Auto-generated method stub - return ERROR_PATH; - } } \ No newline at end of file diff --git a/novel-admin/src/main/java/com/java2nb/common/utils/JSONUtils.java b/novel-admin/src/main/java/com/java2nb/common/utils/JSONUtils.java index 4575bd4..7fc193f 100644 --- a/novel-admin/src/main/java/com/java2nb/common/utils/JSONUtils.java +++ b/novel-admin/src/main/java/com/java2nb/common/utils/JSONUtils.java @@ -1,6 +1,5 @@ package com.java2nb.common.utils; -import com.alibaba.druid.util.StringUtils; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; @@ -8,79 +7,80 @@ import java.util.HashMap; import java.util.Map; public class JSONUtils { - /** - * Bean对象转JSON - * - * @param object - * @param dataFormatString - * @return - */ - public static String beanToJson(Object object, String dataFormatString) { - if (object != null) { - if (StringUtils.isEmpty(dataFormatString)) { - return JSONObject.toJSONString(object); - } - return JSON.toJSONStringWithDateFormat(object, dataFormatString); - } else { - return null; - } - } - /** - * Bean对象转JSON - * - * @param object - * @return - */ - public static String beanToJson(Object object) { - if (object != null) { - return JSON.toJSONString(object); - } else { - return null; - } - } + /** + * Bean对象转JSON + * + * @param object + * @param dataFormatString + * @return + */ + public static String beanToJson(Object object, String dataFormatString) { + if (object != null) { + if (StringUtils.isEmpty(dataFormatString)) { + return JSONObject.toJSONString(object); + } + return JSON.toJSONStringWithDateFormat(object, dataFormatString); + } else { + return null; + } + } - /** - * String转JSON字符串 - * - * @param key - * @param value - * @return - */ - public static String stringToJsonByFastjson(String key, String value) { - if (StringUtils.isEmpty(key) || StringUtils.isEmpty(value)) { - return null; - } - Map map = new HashMap(16); - map.put(key, value); - return beanToJson(map, null); - } + /** + * Bean对象转JSON + * + * @param object + * @return + */ + public static String beanToJson(Object object) { + if (object != null) { + return JSON.toJSONString(object); + } else { + return null; + } + } - /** - * 将json字符串转换成对象 - * - * @param json - * @param clazz - * @return - */ - public static Object jsonToBean(String json, Object clazz) { - if (StringUtils.isEmpty(json) || clazz == null) { - return null; - } - return JSON.parseObject(json, clazz.getClass()); - } + /** + * String转JSON字符串 + * + * @param key + * @param value + * @return + */ + public static String stringToJsonByFastjson(String key, String value) { + if (StringUtils.isEmpty(key) || StringUtils.isEmpty(value)) { + return null; + } + Map map = new HashMap(16); + map.put(key, value); + return beanToJson(map, null); + } - /** - * json字符串转map - * - * @param json - * @return - */ - @SuppressWarnings("unchecked") - public static Map jsonToMap(String json) { - if (StringUtils.isEmpty(json)) { - return null; - } - return JSON.parseObject(json, Map.class); - } + /** + * 将json字符串转换成对象 + * + * @param json + * @param clazz + * @return + */ + public static Object jsonToBean(String json, Object clazz) { + if (StringUtils.isEmpty(json) || clazz == null) { + return null; + } + return JSON.parseObject(json, clazz.getClass()); + } + + /** + * json字符串转map + * + * @param json + * @return + */ + @SuppressWarnings("unchecked") + public static Map jsonToMap(String json) { + if (StringUtils.isEmpty(json)) { + return null; + } + return JSON.parseObject(json, Map.class); + } } diff --git a/novel-admin/src/main/resources/application-dev.yml b/novel-admin/src/main/resources/application-dev.yml index 0ea1099..0efd72e 100644 --- a/novel-admin/src/main/resources/application-dev.yml +++ b/novel-admin/src/main/resources/application-dev.yml @@ -7,92 +7,11 @@ logging: root: info com.java2nb: debug spring: - datasource: - type: com.alibaba.druid.pool.DruidDataSource - driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true - username: root - password: test123456 - #password: - initialSize: 1 - minIdle: 3 - maxActive: 20 - # 配置获取连接等待超时的时间 - maxWait: 60000 - # 配置间隔多久才进行一次检测,检测需要关闭的空闲连接,单位是毫秒 - timeBetweenEvictionRunsMillis: 60000 - # 配置一个连接在池中最小生存的时间,单位是毫秒 - minEvictableIdleTimeMillis: 30000 - validationQuery: select 'x' - testWhileIdle: true - testOnBorrow: false - testOnReturn: false - # 打开PSCache,并且指定每个连接上PSCache的大小 - poolPreparedStatements: true - maxPoolPreparedStatementPerConnectionSize: 20 - # 配置监控统计拦截的filters,去掉后监控界面sql无法统计,'wall'用于防火墙 - filters: stat,slf4j - # 通过connectProperties属性来打开mergeSql功能;慢SQL记录 - connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 - # 合并多个DruidDataSource的监控数据 - #useGlobalDataSourceStat: true redis: host: 127.0.0.1 port: 6379 password: test123456 # 连接超时时间(毫秒) timeout: 10000 - jedis: - pool: - # 连接池中的最大空闲连接 - max-idle: 8 - # 连接池中的最小空闲连接 - min-idle: 10 - # 连接池最大连接数(使用负值表示没有限制) - max-active: 100 - # 连接池最大阻塞等待时间(使用负值表示没有限制) - max-wait: -1 - -####使用shardingJdbc时, -####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误 -##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR - -sharding: - jdbc: - datasource: - names: ds0,ds1 - ds0: - type: com.zaxxer.hikari.HikariDataSource - driver-class-name: com.mysql.cj.jdbc.Driver - jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - ds1: - type: com.alibaba.druid.pool.DruidDataSource - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://localhost:3306/information_schema?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - config: - sharding: - props: - sql.show: true - tables: - book_content: #book_content表 - key-generator-column-name: id #主键 - actual-data-nodes: ds${0}.book_content${0..9} #数据节点 - # database-strategy: #分库策略 - # inline: - # sharding-column: book_id - # algorithm-expression: ds${book_id % 10} - table-strategy: #分表策略 - inline: - shardingColumn: index_id - algorithm-expression: book_content${index_id % 10} - tables: - actual-data-nodes: ds${1}.tables - columns: - actual-data-nodes: ds${1}.columns - default-data-source-name: ds0 \ No newline at end of file diff --git a/novel-admin/src/main/resources/application-prod.yml b/novel-admin/src/main/resources/application-prod.yml index bb63df2..0fab20d 100644 --- a/novel-admin/src/main/resources/application-prod.yml +++ b/novel-admin/src/main/resources/application-prod.yml @@ -7,86 +7,11 @@ logging: root: error com.java2nb: error spring: - datasource: - type: com.alibaba.druid.pool.DruidDataSource - driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true - username: root - password: test123456 - #password: - initialSize: 1 - minIdle: 3 - maxActive: 20 - # 配置获取连接等待超时的时间 - maxWait: 60000 - # 配置间隔多久才进行一次检测,检测需要关闭的空闲连接,单位是毫秒 - timeBetweenEvictionRunsMillis: 60000 - # 配置一个连接在池中最小生存的时间,单位是毫秒 - minEvictableIdleTimeMillis: 30000 - validationQuery: select 'x' - testWhileIdle: true - testOnBorrow: false - testOnReturn: false - # 打开PSCache,并且指定每个连接上PSCache的大小 - poolPreparedStatements: true - maxPoolPreparedStatementPerConnectionSize: 20 - # 配置监控统计拦截的filters,去掉后监控界面sql无法统计,'wall'用于防火墙 - filters: stat,slf4j - # 通过connectProperties属性来打开mergeSql功能;慢SQL记录 - connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 - # 合并多个DruidDataSource的监控数据 - #useGlobalDataSourceStat: true redis: host: 127.0.0.1 port: 6379 - password: test + password: test123456 # 连接超时时间(毫秒) timeout: 10000 - jedis: - pool: - # 连接池中的最大空闲连接 - max-idle: 8 - # 连接池中的最小空闲连接 - min-idle: 10 - # 连接池最大连接数(使用负值表示没有限制) - max-active: 100 - # 连接池最大阻塞等待时间(使用负值表示没有限制) - max-wait: -1 -####使用shardingJdbc时, -####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误 -##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR - -sharding: - jdbc: - datasource: - names: ds0 #,ds1 - ds0: - type: com.zaxxer.hikari.HikariDataSource - driver-class-name: com.mysql.cj.jdbc.Driver - jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - # ds1: - # type: com.alibaba.druid.pool.DruidDataSource - # driver-class-name: com.mysql.jdbc.Driver - # url: jdbc:mysql://localhost:3306/novel_plus2 - # username: root - # password: test123456 - config: - sharding: - props: - sql.show: true - tables: - book_content: #book_content表 - key-generator-column-name: id #主键 - actual-data-nodes: ds${0}.book_content${0..9} #数据节点 - # database-strategy: #分库策略 - # inline: - # sharding-column: book_id - # algorithm-expression: ds${book_id % 10} - table-strategy: #分表策略 - inline: - shardingColumn: index_id - algorithm-expression: book_content${index_id % 10} diff --git a/novel-admin/src/main/resources/application.yml b/novel-admin/src/main/resources/application.yml index 7c2fd02..16fae56 100644 --- a/novel-admin/src/main/resources/application.yml +++ b/novel-admin/src/main/resources/application.yml @@ -9,6 +9,9 @@ server: # basic: # enabled: false spring: + datasource: + driverClassName: org.apache.shardingsphere.driver.ShardingSphereDriver + url: jdbc:shardingsphere:absolutepath:${user.dir}/config/shardingsphere-jdbc.yml thymeleaf: mode: LEGACYHTML5 cache: false @@ -23,10 +26,7 @@ spring: multipart: max-file-size: 100MB max-request-size: 100MB - - devtools: - restart: - enabled: true + main: allow-bean-definition-overriding: true @@ -36,9 +36,6 @@ mybatis: map-underscore-to-camel-case: true mapper-locations: mybatis/**/*Mapper.xml typeAliasesPackage: com.java2nb.**.domain -#[弃用]配置缓存和session存储方式,默认ehcache,可选redis,[弃用]调整至 spring cache type【shiro.用户,权限,session,spring.cache通用】 -#[弃用]cacheType: ehcache - logging: config: classpath:logback-boot.xml diff --git a/novel-admin/src/main/resources/favicon.ico b/novel-admin/src/main/resources/static/favicon.ico similarity index 100% rename from novel-admin/src/main/resources/favicon.ico rename to novel-admin/src/main/resources/static/favicon.ico diff --git a/novel-admin/src/main/resources/static/js/appjs/novel/websiteInfo/edit.js b/novel-admin/src/main/resources/static/js/appjs/novel/websiteInfo/edit.js index b3384bd..0a692f2 100644 --- a/novel-admin/src/main/resources/static/js/appjs/novel/websiteInfo/edit.js +++ b/novel-admin/src/main/resources/static/js/appjs/novel/websiteInfo/edit.js @@ -79,7 +79,7 @@ function update() { }, success: function (data) { if (data.code == 0) { - layer.msg("操作成功"); + layer.msg("操作成功,重启 novel-front 后生效"); } else { layer.alert(data.msg) } diff --git a/novel-admin/src/main/resources/templates/common/file/file.html b/novel-admin/src/main/resources/templates/common/file/file.html index fa0de95..9efd53f 100644 --- a/novel-admin/src/main/resources/templates/common/file/file.html +++ b/novel-admin/src/main/resources/templates/common/file/file.html @@ -6,7 +6,7 @@ 小说精品屋 - 文件管理器 - + diff --git a/novel-admin/src/main/resources/templates/error/403.html b/novel-admin/src/main/resources/templates/error/403.html index 7738e91..2e98ace 100644 --- a/novel-admin/src/main/resources/templates/error/403.html +++ b/novel-admin/src/main/resources/templates/error/403.html @@ -6,7 +6,7 @@ 403 页面 - + diff --git a/novel-admin/src/main/resources/templates/error/404.html b/novel-admin/src/main/resources/templates/error/404.html index 55cd586..babef5d 100644 --- a/novel-admin/src/main/resources/templates/error/404.html +++ b/novel-admin/src/main/resources/templates/error/404.html @@ -10,7 +10,7 @@ - + diff --git a/novel-admin/src/main/resources/templates/error/500.html b/novel-admin/src/main/resources/templates/error/500.html index 194f536..61981d8 100644 --- a/novel-admin/src/main/resources/templates/error/500.html +++ b/novel-admin/src/main/resources/templates/error/500.html @@ -11,7 +11,7 @@ - + diff --git a/novel-admin/src/main/resources/templates/error/error.html b/novel-admin/src/main/resources/templates/error/error.html index 802339f..9db6e11 100644 --- a/novel-admin/src/main/resources/templates/error/error.html +++ b/novel-admin/src/main/resources/templates/error/error.html @@ -6,7 +6,7 @@ 500错误 - + diff --git a/novel-admin/src/main/resources/templates/include.html b/novel-admin/src/main/resources/templates/include.html index 9bd4ce7..90be390 100644 --- a/novel-admin/src/main/resources/templates/include.html +++ b/novel-admin/src/main/resources/templates/include.html @@ -4,7 +4,7 @@ - + - + diff --git a/novel-admin/src/main/resources/templates/login.html b/novel-admin/src/main/resources/templates/login.html index 2f09be2..f57eb80 100644 --- a/novel-admin/src/main/resources/templates/login.html +++ b/novel-admin/src/main/resources/templates/login.html @@ -2,7 +2,7 @@ - 后台管理-登陆 + 后台管理-登录 diff --git a/novel-admin/src/main/resources/templates/system/menu/edit.html b/novel-admin/src/main/resources/templates/system/menu/edit.html index a7bcfba..6c42c89 100644 --- a/novel-admin/src/main/resources/templates/system/menu/edit.html +++ b/novel-admin/src/main/resources/templates/system/menu/edit.html @@ -11,7 +11,7 @@ - + diff --git a/novel-admin/src/main/resources/templates/system/user/include.html b/novel-admin/src/main/resources/templates/system/user/include.html index 6498fec..7dec980 100644 --- a/novel-admin/src/main/resources/templates/system/user/include.html +++ b/novel-admin/src/main/resources/templates/system/user/include.html @@ -4,7 +4,7 @@ - + diff --git a/novel-common/pom.xml b/novel-common/pom.xml index 0f7bde4..b21ca1a 100644 --- a/novel-common/pom.xml +++ b/novel-common/pom.xml @@ -5,7 +5,7 @@ novel com.java2nb - 4.3.0 + 5.2.0 4.0.0 @@ -52,18 +52,16 @@ - - + - io.shardingsphere - sharding-jdbc-spring-boot-starter - ${sharding.jdbc.version} + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere-jdbc.version} - - io.shardingsphere - sharding-jdbc-spring-namespace - ${sharding.jdbc.version} + com.h2database + h2 + runtime @@ -73,11 +71,6 @@ pagehelper-spring-boot-starter ${pagehelper.version} - - com.cuisongliu - orderbyhelper-spring-boot-starter - ${orderbyhelper.version} - org.apache.commons @@ -86,15 +79,8 @@ - org.apache.httpcomponents - httpclient - 4.5.14 - - - commons-logging - commons-logging - - + org.apache.httpcomponents.client5 + httpclient5 @@ -117,6 +103,12 @@ true + + + org.hibernate.validator + hibernate-validator + + io.github.xxyopen xxy-model @@ -138,5 +130,29 @@ + + + + maven-antrun-plugin + 1.8 + + + package + + run + + + + + + + + + + + + + + \ No newline at end of file diff --git a/novel-common/src/main/java/com/java2nb/novel/core/advice/CommonExceptionHandler.java b/novel-common/src/main/java/com/java2nb/novel/core/advice/CommonExceptionHandler.java new file mode 100644 index 0000000..cae9f25 --- /dev/null +++ b/novel-common/src/main/java/com/java2nb/novel/core/advice/CommonExceptionHandler.java @@ -0,0 +1,55 @@ +package com.java2nb.novel.core.advice; + +import io.github.xxyopen.model.resp.RestResult; +import io.github.xxyopen.model.resp.SysResultCode; +import io.github.xxyopen.web.exception.BusinessException; +import jakarta.servlet.http.HttpServletRequest; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.MediaType; +import org.springframework.validation.BindException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.servlet.ModelAndView; + +/** + * 统一异常处理器 + * + * @author xiongxiaoyang + */ +@Slf4j +@RestControllerAdvice +public class CommonExceptionHandler { + + public CommonExceptionHandler() { + } + + @ExceptionHandler({BindException.class}) + public RestResult handlerBindException(BindException e) { + log.error(e.getMessage(), e); + return RestResult.fail(SysResultCode.PARAM_ERROR); + } + + @ExceptionHandler({BusinessException.class}) + public RestResult handlerBusinessException(BusinessException e) { + log.error(e.getMessage(), e); + return RestResult.fail(e.getResultCode()); + } + + @ExceptionHandler(Exception.class) + public Object handleException(HttpServletRequest request, Exception e) { + log.error(e.getMessage(), e); + if (isJsonRequest(request)) { + // 如果是REST请求,返回JSON格式的错误响应 + return RestResult.error(); + } else { + //跳转页面过程中出现异常时统一跳转到404页面 + return new ModelAndView("404"); + } + } + + private boolean isJsonRequest(HttpServletRequest request) { + String acceptHeader = request.getHeader("Accept"); + return acceptHeader != null && acceptHeader.contains(MediaType.APPLICATION_JSON_VALUE); + } + +} \ No newline at end of file diff --git a/novel-common/src/main/java/com/java2nb/novel/core/advice/CustomResponseBodyAdvice.java b/novel-common/src/main/java/com/java2nb/novel/core/advice/CustomResponseBodyAdvice.java new file mode 100644 index 0000000..fa96131 --- /dev/null +++ b/novel-common/src/main/java/com/java2nb/novel/core/advice/CustomResponseBodyAdvice.java @@ -0,0 +1,53 @@ +package com.java2nb.novel.core.advice; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import org.springframework.core.MethodParameter; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; + +import java.util.Objects; + +/** + * 在对 RestController 返回对象 json 序列化时,将所有 Long 类型转为 String 类型返回,避免前端数据精度丢失的问题 + * 取代 spring.jackson.generator.write-numbers-as-strings=true 配置,避免影响全局的 ObjectMapper + * + * @author xiongxiaoyang + * */ +@RestControllerAdvice +public class CustomResponseBodyAdvice implements ResponseBodyAdvice { + + private final ObjectMapper customObjectMapper; + + public CustomResponseBodyAdvice(Jackson2ObjectMapperBuilder builder) { + customObjectMapper = builder.createXmlMapper(false).build(); + SimpleModule simpleModule = new SimpleModule(); + simpleModule.addSerializer(Long.class, ToStringSerializer.instance); + simpleModule.addSerializer(Long.TYPE, ToStringSerializer.instance); + customObjectMapper.registerModule(simpleModule); + } + + @Override + public boolean supports(MethodParameter returnType, Class> converterType) { + // 返回 true 表示对所有 Controller 的响应都生效 + return true; + } + + @Override + public Object beforeBodyWrite(Object body, MethodParameter returnType, MediaType selectedContentType, Class> selectedConverterType, ServerHttpRequest request, ServerHttpResponse response) { + // 使用自定义的 ObjectMapper 序列化响应体 + if(Objects.nonNull(body)) { + return customObjectMapper.valueToTree(body); + }else{ + return null; + } + } + +} + diff --git a/novel-common/src/main/java/com/java2nb/novel/core/advice/PageExceptionHandler.java b/novel-common/src/main/java/com/java2nb/novel/core/advice/PageExceptionHandler.java deleted file mode 100644 index 85041fd..0000000 --- a/novel-common/src/main/java/com/java2nb/novel/core/advice/PageExceptionHandler.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.java2nb.novel.core.advice; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; - -/** - * 页面异常处理器 - * - * @author 11797 - */ -@Slf4j -@ControllerAdvice(basePackages = "com.java2nb.novel.controller.page") -public class PageExceptionHandler { - - - /** - * 处理所有异常 - */ - @ExceptionHandler(Exception.class) - public String handlerException(Exception e) { - log.error(e.getMessage(), e); - //跳转页面过程中出现异常时统一跳转到404页面 - return "404"; - } -} diff --git a/novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java b/novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java index 750c53f..d1b5811 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java @@ -41,11 +41,6 @@ public interface CacheKey { * */ String TEMPLATE_DIR_KEY = "templateDirKey";; - /** - * 正在运行的爬虫线程存储KEY前缀 - * */ - String RUNNING_CRAWL_THREAD_KEY_PREFIX = "runningCrawlTreadDataKeyPrefix"; - /** * 上一次搜索引擎更新的时间 * */ @@ -69,4 +64,8 @@ public interface CacheKey { * 测试爬虫规则缓存 */ String BOOK_TEST_PARSE = "testParse"; + /** + * AI生成图片 + * */ + String AI_GEN_PIC = "aiGenPic"; } diff --git a/novel-common/src/main/java/com/java2nb/novel/core/config/HttpProxyProperties.java b/novel-common/src/main/java/com/java2nb/novel/core/config/HttpProxyProperties.java index 140f298..d128c8e 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/config/HttpProxyProperties.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/config/HttpProxyProperties.java @@ -19,4 +19,8 @@ public class HttpProxyProperties { private Integer port; + private String username; + + private String password; + } diff --git a/novel-common/src/main/java/com/java2nb/novel/core/enums/ResponseStatus.java b/novel-common/src/main/java/com/java2nb/novel/core/enums/ResponseStatus.java index 1fce47a..d51596f 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/enums/ResponseStatus.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/enums/ResponseStatus.java @@ -18,7 +18,7 @@ public enum ResponseStatus implements IResultCode { /** * 用户相关错误 * */ - NO_LOGIN(1001, "未登录或登陆失效!"), + NO_LOGIN(1001, "未登录或登录失效!"), VEL_CODE_ERROR(1002, "验证码错误!"), USERNAME_EXIST(1003,"该手机号已注册!"), USERNAME_PASS_ERROR(1004,"手机号或密码错误!"), diff --git a/novel-common/src/main/java/com/java2nb/novel/core/utils/DateUtil.java b/novel-common/src/main/java/com/java2nb/novel/core/utils/DateUtil.java index c41bb36..2d09f9c 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/utils/DateUtil.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/utils/DateUtil.java @@ -106,6 +106,47 @@ public class DateUtil { } + /** + * 将日期格式化成"多久之前"的格式 + * */ + public static String formatTimeAgo(Date date){ + if (date == null) { + return null; + } + + long now = new Date().getTime(); + long then = date.getTime(); + + long diff = now - then; + + if (diff < 0) { + // 未来时间 + DateUtil.formatDate(date, DateUtil.DATE_TIME_PATTERN); + } + + long seconds = diff / 1000; + long minutes = seconds / 60; + long hours = minutes / 60; + long days = hours / 24; + long months = days / 30; + long years = months / 12; + + if (seconds < 60) { + return "刚刚"; + } else if (minutes < 60) { + return minutes + "分钟前"; + } else if (hours < 24) { + return hours + "小时前"; + } else if (days < 30) { + return days + "天前"; + } else if (months < 12) { + return months + "个月前"; + } else { + return years + "年前"; + } + } + + public static void main(String[] args) { System.out.println(formatDate(getYesterday(),DATE_TIME_PATTERN)); System.out.println(formatDate(getDateStartTime(getYesterday()),DATE_TIME_PATTERN)); diff --git a/novel-common/src/main/java/com/java2nb/novel/core/utils/FileUtil.java b/novel-common/src/main/java/com/java2nb/novel/core/utils/FileUtil.java index f423fe0..656fe8f 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/utils/FileUtil.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/utils/FileUtil.java @@ -5,7 +5,7 @@ import lombok.SneakyThrows; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.Charsets; -import org.apache.http.client.utils.DateUtils; +import org.apache.hc.client5.http.utils.DateUtils; import org.springframework.core.io.Resource; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; @@ -14,7 +14,15 @@ import org.springframework.http.ResponseEntity; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.Date; import java.util.Objects; @@ -37,10 +45,13 @@ public class FileUtil { //本地图片保存 HttpHeaders headers = new HttpHeaders(); HttpEntity requestEntity = new HttpEntity<>(null, headers); - ResponseEntity resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1.name()).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); + ResponseEntity resEntity = RestTemplates.newInstance(Charsets.ISO_8859_1.name()) + .exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); input = Objects.requireNonNull(resEntity.getBody()).getInputStream(); Date currentDate = new Date(); - picSrc = visitPrefix + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd") + "/" + picSrc = + visitPrefix + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + + "/" + DateUtils.formatDate(currentDate, "dd") + "/" + UUIDUtil.getUUID32() + picSrc.substring(picSrc.lastIndexOf(".")); File picFile = new File(picSavePath + picSrc); @@ -67,7 +78,6 @@ public class FileUtil { closeStream(input, out); } - return picSrc; } @@ -120,5 +130,23 @@ public class FileUtil { } + /** + * 下载文件 + * + * @param downloadUrl 下载的URL + * @param savePath 保存的路径 + */ + @SneakyThrows + public void downloadFile(String downloadUrl, String savePath) { + Path path = Paths.get(savePath); + Path parentPath = path.getParent(); + if (Files.notExists(parentPath)) { + Files.createDirectories(parentPath); + } + URL url = new URL(downloadUrl); + try (InputStream in = url.openStream()) { + Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + } + } } diff --git a/novel-common/src/main/java/com/java2nb/novel/core/utils/HttpUtil.java b/novel-common/src/main/java/com/java2nb/novel/core/utils/HttpUtil.java index d1d65a1..c9a3d7a 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/utils/HttpUtil.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/utils/HttpUtil.java @@ -1,47 +1,52 @@ package com.java2nb.novel.core.utils; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.*; import org.springframework.web.client.RestTemplate; +import java.nio.charset.Charset; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + /** * @author Administrator */ +@Slf4j public class HttpUtil { - private static RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8"); + private static final String DEFAULT_CHARSET = "utf-8"; + private static final Map REST_TEMPLATE_MAP = new ConcurrentHashMap<>(); - public static String getByHttpClient(String url) { + public static String getByHttpClientWithChrome(String url, String charset) { + log.debug("Get url:{}", url); + if (!Charset.isSupported(charset)) { + log.error("字符编码{}无效!", charset); + return null; + } + RestTemplate restTemplate = REST_TEMPLATE_MAP.computeIfAbsent(charset, + k -> RestTemplates.newInstance(charset)); try { - - ResponseEntity forEntity = restTemplate.getForEntity(url, String.class); + HttpHeaders headers = new HttpHeaders(); + headers.add("user-agent", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36"); + HttpEntity requestEntity = new HttpEntity<>(null, headers); + ResponseEntity forEntity = restTemplate.exchange(url, HttpMethod.GET, requestEntity, + String.class); + log.debug("Response code:{}", forEntity.getStatusCode()); if (forEntity.getStatusCode() == HttpStatus.OK) { return forEntity.getBody(); } else { return null; } } catch (Exception e) { - e.printStackTrace(); + log.error(e.getMessage(), e); return null; } } public static String getByHttpClientWithChrome(String url) { - try { - - HttpHeaders headers = new HttpHeaders(); - headers.add("user-agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36"); - HttpEntity requestEntity = new HttpEntity<>(null, headers); - ResponseEntity forEntity = restTemplate.exchange(url.toString(), HttpMethod.GET, requestEntity, String.class); - - if (forEntity.getStatusCode() == HttpStatus.OK) { - return forEntity.getBody(); - } else { - return null; - } - } catch (Exception e) { - e.printStackTrace(); - return null; - } + return getByHttpClientWithChrome(url, DEFAULT_CHARSET); } + } diff --git a/novel-common/src/main/java/com/java2nb/novel/core/utils/IpUtil.java b/novel-common/src/main/java/com/java2nb/novel/core/utils/IpUtil.java index c7f92d2..53f836d 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/utils/IpUtil.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/utils/IpUtil.java @@ -1,11 +1,21 @@ package com.java2nb.novel.core.utils; -import javax.servlet.http.HttpServletRequest; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.http.HttpServletRequest; +import lombok.extern.slf4j.Slf4j; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +@Slf4j public class IpUtil { /** * 获取真实IP + * * @param request 请求体 * @return 真实IP */ @@ -31,4 +41,27 @@ public class IpUtil { } return ip; } + + /** + * 获取本机公网IP + */ + public static String getPublicIP() { + try { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("https://httpbin.org/ip")) + .GET() + .timeout(Duration.ofSeconds(5)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() == 200) { + return new ObjectMapper().readTree(response.body()).get("origin").asText(); + } + } catch (Exception e) { + log.error("获取本机公网IP异常", e); + } + return null; + } } diff --git a/novel-common/src/main/java/com/java2nb/novel/core/utils/RestTemplateUtil.java b/novel-common/src/main/java/com/java2nb/novel/core/utils/RestTemplates.java similarity index 62% rename from novel-common/src/main/java/com/java2nb/novel/core/utils/RestTemplateUtil.java rename to novel-common/src/main/java/com/java2nb/novel/core/utils/RestTemplates.java index d4d2684..b7de405 100644 --- a/novel-common/src/main/java/com/java2nb/novel/core/utils/RestTemplateUtil.java +++ b/novel-common/src/main/java/com/java2nb/novel/core/utils/RestTemplates.java @@ -2,17 +2,22 @@ package com.java2nb.novel.core.utils; import com.java2nb.novel.core.config.HttpProxyProperties; import lombok.SneakyThrows; -import org.apache.http.HttpHost; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.commons.lang3.StringUtils; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.client5.http.socket.ConnectionSocketFactory; +import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory; +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.config.Registry; +import org.apache.hc.core5.http.config.RegistryBuilder; +import org.apache.hc.core5.ssl.SSLContexts; +import org.apache.hc.core5.ssl.TrustStrategy; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.StringHttpMessageConverter; @@ -26,21 +31,21 @@ import java.util.List; import java.util.Objects; @Component -public class RestTemplateUtil { +public class RestTemplates { private static HttpProxyProperties httpProxyProperties; - RestTemplateUtil(HttpProxyProperties properties) { + RestTemplates(HttpProxyProperties properties) { httpProxyProperties = properties; } @SneakyThrows - public static RestTemplate getInstance(String charset) { + public static RestTemplate newInstance(String charset) { TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true; //忽略证书 - SSLContext sslContext = org.apache.http.ssl.SSLContexts.custom() + SSLContext sslContext = SSLContexts.custom() .loadTrustMaterial(null, acceptingTrustStrategy) .build(); @@ -61,6 +66,15 @@ public class RestTemplateUtil { if (Objects.nonNull(httpProxyProperties) && Boolean.TRUE.equals(httpProxyProperties.getEnabled())) { HttpHost proxy = new HttpHost(httpProxyProperties.getIp(), httpProxyProperties.getPort()); clientBuilder.setProxy(proxy); + if (StringUtils.isNotBlank(httpProxyProperties.getUsername()) && StringUtils.isNotBlank( + httpProxyProperties.getPassword())) { + // 创建CredentialsProvider实例并添加代理认证信息 + BasicCredentialsProvider provider = new BasicCredentialsProvider(); + UsernamePasswordCredentials credentials = new UsernamePasswordCredentials( + httpProxyProperties.getUsername(), httpProxyProperties.getPassword().toCharArray()); + provider.setCredentials(new AuthScope(null, -1), credentials); + clientBuilder.setDefaultCredentialsProvider(provider); + } } CloseableHttpClient httpClient = clientBuilder.setConnectionManager(connectionManager) .build(); diff --git a/novel-common/src/main/java/com/java2nb/novel/core/utils/SpringUtil.java b/novel-common/src/main/java/com/java2nb/novel/core/utils/SpringUtil.java new file mode 100644 index 0000000..5c7ea6d --- /dev/null +++ b/novel-common/src/main/java/com/java2nb/novel/core/utils/SpringUtil.java @@ -0,0 +1,37 @@ +package com.java2nb.novel.core.utils; + +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +@Component +public class SpringUtil implements ApplicationContextAware { + private static ApplicationContext applicationContext; + + public SpringUtil() { + } + + public void setApplicationContext(ApplicationContext applicationContext) { + if (SpringUtil.applicationContext == null) { + SpringUtil.applicationContext = applicationContext; + } + + } + + public static ApplicationContext getApplicationContext() { + return applicationContext; + } + + public static Object getBean(String name) { + return getApplicationContext().getBean(name); + } + + public static T getBean(Class clazz) { + return (T)getApplicationContext().getBean(clazz); + } + + public static T getBean(String name, Class clazz) { + return (T)getApplicationContext().getBean(name, clazz); + } +} + diff --git a/novel-common/src/main/java/com/java2nb/novel/entity/BookComment.java b/novel-common/src/main/java/com/java2nb/novel/entity/BookComment.java index 98c48b2..095a716 100644 --- a/novel-common/src/main/java/com/java2nb/novel/entity/BookComment.java +++ b/novel-common/src/main/java/com/java2nb/novel/entity/BookComment.java @@ -14,6 +14,9 @@ public class BookComment { @Generated("org.mybatis.generator.api.MyBatisGenerator") private String commentContent; + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private String location; + @Generated("org.mybatis.generator.api.MyBatisGenerator") private Integer replyCount; @@ -56,6 +59,16 @@ public class BookComment { this.commentContent = commentContent == null ? null : commentContent.trim(); } + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public String getLocation() { + return location; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setLocation(String location) { + this.location = location == null ? null : location.trim(); + } + @Generated("org.mybatis.generator.api.MyBatisGenerator") public Integer getReplyCount() { return replyCount; diff --git a/novel-common/src/main/java/com/java2nb/novel/entity/BookCommentReply.java b/novel-common/src/main/java/com/java2nb/novel/entity/BookCommentReply.java new file mode 100644 index 0000000..77ea87b --- /dev/null +++ b/novel-common/src/main/java/com/java2nb/novel/entity/BookCommentReply.java @@ -0,0 +1,97 @@ +package com.java2nb.novel.entity; + +import java.util.Date; +import javax.annotation.Generated; + +public class BookCommentReply { + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private Long id; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private Long commentId; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private String replyContent; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private String location; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private Byte auditStatus; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private Date createTime; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private Long createUserId; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public Long getId() { + return id; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setId(Long id) { + this.id = id; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public Long getCommentId() { + return commentId; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setCommentId(Long commentId) { + this.commentId = commentId; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public String getReplyContent() { + return replyContent; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setReplyContent(String replyContent) { + this.replyContent = replyContent == null ? null : replyContent.trim(); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public String getLocation() { + return location; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setLocation(String location) { + this.location = location == null ? null : location.trim(); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public Byte getAuditStatus() { + return auditStatus; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setAuditStatus(Byte auditStatus) { + this.auditStatus = auditStatus; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public Date getCreateTime() { + return createTime; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public Long getCreateUserId() { + return createUserId; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setCreateUserId(Long createUserId) { + this.createUserId = createUserId; + } +} \ No newline at end of file diff --git a/novel-common/src/main/java/com/java2nb/novel/entity/CrawlSingleTask.java b/novel-common/src/main/java/com/java2nb/novel/entity/CrawlSingleTask.java index 0f72b05..274d9dc 100644 --- a/novel-common/src/main/java/com/java2nb/novel/entity/CrawlSingleTask.java +++ b/novel-common/src/main/java/com/java2nb/novel/entity/CrawlSingleTask.java @@ -31,6 +31,9 @@ public class CrawlSingleTask { @Generated("org.mybatis.generator.api.MyBatisGenerator") private Byte excCount; + @Generated("org.mybatis.generator.api.MyBatisGenerator") + private Integer crawlChapters; + @Generated("org.mybatis.generator.api.MyBatisGenerator") private Date createTime; @@ -124,6 +127,16 @@ public class CrawlSingleTask { this.excCount = excCount; } + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public Integer getCrawlChapters() { + return crawlChapters; + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public void setCrawlChapters(Integer crawlChapters) { + this.crawlChapters = crawlChapters; + } + @Generated("org.mybatis.generator.api.MyBatisGenerator") public Date getCreateTime() { return createTime; diff --git a/novel-common/src/main/java/com/java2nb/novel/entity/User.java b/novel-common/src/main/java/com/java2nb/novel/entity/User.java index a3a1d5a..7da42eb 100644 --- a/novel-common/src/main/java/com/java2nb/novel/entity/User.java +++ b/novel-common/src/main/java/com/java2nb/novel/entity/User.java @@ -2,9 +2,9 @@ package com.java2nb.novel.entity; import io.github.xxyopen.web.valid.AddGroup; import io.github.xxyopen.web.valid.UpdateGroup; +import jakarta.validation.constraints.*; import javax.annotation.Generated; -import javax.validation.constraints.*; import java.util.Date; public class User { diff --git a/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentDynamicSqlSupport.java b/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentDynamicSqlSupport.java index cc0fd4a..8fe3ac0 100644 --- a/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentDynamicSqlSupport.java +++ b/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentDynamicSqlSupport.java @@ -19,6 +19,9 @@ public final class BookCommentDynamicSqlSupport { @Generated("org.mybatis.generator.api.MyBatisGenerator") public static final SqlColumn commentContent = bookComment.commentContent; + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn location = bookComment.location; + @Generated("org.mybatis.generator.api.MyBatisGenerator") public static final SqlColumn replyCount = bookComment.replyCount; @@ -39,6 +42,8 @@ public final class BookCommentDynamicSqlSupport { public final SqlColumn commentContent = column("comment_content", JDBCType.VARCHAR); + public final SqlColumn location = column("location", JDBCType.VARCHAR); + public final SqlColumn replyCount = column("reply_count", JDBCType.INTEGER); public final SqlColumn auditStatus = column("audit_status", JDBCType.TINYINT); diff --git a/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentMapper.java b/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentMapper.java index 2ba063e..9d93f76 100644 --- a/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentMapper.java +++ b/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentMapper.java @@ -29,7 +29,7 @@ import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo; @Mapper public interface BookCommentMapper { @Generated("org.mybatis.generator.api.MyBatisGenerator") - BasicColumn[] selectList = BasicColumn.columnList(id, bookId, commentContent, replyCount, auditStatus, createTime, createUserId); + BasicColumn[] selectList = BasicColumn.columnList(id, bookId, commentContent, location, replyCount, auditStatus, createTime, createUserId); @Generated("org.mybatis.generator.api.MyBatisGenerator") @SelectProvider(type=SqlProviderAdapter.class, method="select") @@ -58,6 +58,7 @@ public interface BookCommentMapper { @Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true), @Result(column="book_id", property="bookId", jdbcType=JdbcType.BIGINT), @Result(column="comment_content", property="commentContent", jdbcType=JdbcType.VARCHAR), + @Result(column="location", property="location", jdbcType=JdbcType.VARCHAR), @Result(column="reply_count", property="replyCount", jdbcType=JdbcType.INTEGER), @Result(column="audit_status", property="auditStatus", jdbcType=JdbcType.TINYINT), @Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP), @@ -92,6 +93,7 @@ public interface BookCommentMapper { c.map(id).toProperty("id") .map(bookId).toProperty("bookId") .map(commentContent).toProperty("commentContent") + .map(location).toProperty("location") .map(replyCount).toProperty("replyCount") .map(auditStatus).toProperty("auditStatus") .map(createTime).toProperty("createTime") @@ -105,6 +107,7 @@ public interface BookCommentMapper { c.map(id).toProperty("id") .map(bookId).toProperty("bookId") .map(commentContent).toProperty("commentContent") + .map(location).toProperty("location") .map(replyCount).toProperty("replyCount") .map(auditStatus).toProperty("auditStatus") .map(createTime).toProperty("createTime") @@ -118,6 +121,7 @@ public interface BookCommentMapper { c.map(id).toPropertyWhenPresent("id", record::getId) .map(bookId).toPropertyWhenPresent("bookId", record::getBookId) .map(commentContent).toPropertyWhenPresent("commentContent", record::getCommentContent) + .map(location).toPropertyWhenPresent("location", record::getLocation) .map(replyCount).toPropertyWhenPresent("replyCount", record::getReplyCount) .map(auditStatus).toPropertyWhenPresent("auditStatus", record::getAuditStatus) .map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime) @@ -157,6 +161,7 @@ public interface BookCommentMapper { return dsl.set(id).equalTo(record::getId) .set(bookId).equalTo(record::getBookId) .set(commentContent).equalTo(record::getCommentContent) + .set(location).equalTo(record::getLocation) .set(replyCount).equalTo(record::getReplyCount) .set(auditStatus).equalTo(record::getAuditStatus) .set(createTime).equalTo(record::getCreateTime) @@ -168,6 +173,7 @@ public interface BookCommentMapper { return dsl.set(id).equalToWhenPresent(record::getId) .set(bookId).equalToWhenPresent(record::getBookId) .set(commentContent).equalToWhenPresent(record::getCommentContent) + .set(location).equalToWhenPresent(record::getLocation) .set(replyCount).equalToWhenPresent(record::getReplyCount) .set(auditStatus).equalToWhenPresent(record::getAuditStatus) .set(createTime).equalToWhenPresent(record::getCreateTime) @@ -179,6 +185,7 @@ public interface BookCommentMapper { return update(c -> c.set(bookId).equalTo(record::getBookId) .set(commentContent).equalTo(record::getCommentContent) + .set(location).equalTo(record::getLocation) .set(replyCount).equalTo(record::getReplyCount) .set(auditStatus).equalTo(record::getAuditStatus) .set(createTime).equalTo(record::getCreateTime) @@ -192,6 +199,7 @@ public interface BookCommentMapper { return update(c -> c.set(bookId).equalToWhenPresent(record::getBookId) .set(commentContent).equalToWhenPresent(record::getCommentContent) + .set(location).equalToWhenPresent(record::getLocation) .set(replyCount).equalToWhenPresent(record::getReplyCount) .set(auditStatus).equalToWhenPresent(record::getAuditStatus) .set(createTime).equalToWhenPresent(record::getCreateTime) diff --git a/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentReplyDynamicSqlSupport.java b/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentReplyDynamicSqlSupport.java new file mode 100644 index 0000000..7373ddd --- /dev/null +++ b/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentReplyDynamicSqlSupport.java @@ -0,0 +1,54 @@ +package com.java2nb.novel.mapper; + +import java.sql.JDBCType; +import java.util.Date; +import javax.annotation.Generated; +import org.mybatis.dynamic.sql.SqlColumn; +import org.mybatis.dynamic.sql.SqlTable; + +public final class BookCommentReplyDynamicSqlSupport { + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final BookCommentReply bookCommentReply = new BookCommentReply(); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn id = bookCommentReply.id; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn commentId = bookCommentReply.commentId; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn replyContent = bookCommentReply.replyContent; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn location = bookCommentReply.location; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn auditStatus = bookCommentReply.auditStatus; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn createTime = bookCommentReply.createTime; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn createUserId = bookCommentReply.createUserId; + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final class BookCommentReply extends SqlTable { + public final SqlColumn id = column("id", JDBCType.BIGINT); + + public final SqlColumn commentId = column("comment_id", JDBCType.BIGINT); + + public final SqlColumn replyContent = column("reply_content", JDBCType.VARCHAR); + + public final SqlColumn location = column("location", JDBCType.VARCHAR); + + public final SqlColumn auditStatus = column("audit_status", JDBCType.TINYINT); + + public final SqlColumn createTime = column("create_time", JDBCType.TIMESTAMP); + + public final SqlColumn createUserId = column("create_user_id", JDBCType.BIGINT); + + public BookCommentReply() { + super("book_comment_reply"); + } + } +} \ No newline at end of file diff --git a/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentReplyMapper.java b/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentReplyMapper.java new file mode 100644 index 0000000..d2ad5b0 --- /dev/null +++ b/novel-common/src/main/java/com/java2nb/novel/mapper/BookCommentReplyMapper.java @@ -0,0 +1,208 @@ +package com.java2nb.novel.mapper; + +import static com.java2nb.novel.mapper.BookCommentReplyDynamicSqlSupport.*; +import static org.mybatis.dynamic.sql.SqlBuilder.*; + +import com.java2nb.novel.entity.BookCommentReply; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import javax.annotation.Generated; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.ResultMap; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; +import org.apache.ibatis.type.JdbcType; +import org.mybatis.dynamic.sql.BasicColumn; +import org.mybatis.dynamic.sql.delete.DeleteDSLCompleter; +import org.mybatis.dynamic.sql.delete.render.DeleteStatementProvider; +import org.mybatis.dynamic.sql.insert.render.InsertStatementProvider; +import org.mybatis.dynamic.sql.insert.render.MultiRowInsertStatementProvider; +import org.mybatis.dynamic.sql.select.CountDSLCompleter; +import org.mybatis.dynamic.sql.select.SelectDSLCompleter; +import org.mybatis.dynamic.sql.select.render.SelectStatementProvider; +import org.mybatis.dynamic.sql.update.UpdateDSL; +import org.mybatis.dynamic.sql.update.UpdateDSLCompleter; +import org.mybatis.dynamic.sql.update.UpdateModel; +import org.mybatis.dynamic.sql.update.render.UpdateStatementProvider; +import org.mybatis.dynamic.sql.util.SqlProviderAdapter; +import org.mybatis.dynamic.sql.util.mybatis3.MyBatis3Utils; + +@Mapper +public interface BookCommentReplyMapper { + @Generated("org.mybatis.generator.api.MyBatisGenerator") + BasicColumn[] selectList = BasicColumn.columnList(id, commentId, replyContent, location, auditStatus, createTime, createUserId); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + @SelectProvider(type=SqlProviderAdapter.class, method="select") + long count(SelectStatementProvider selectStatement); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + @DeleteProvider(type=SqlProviderAdapter.class, method="delete") + int delete(DeleteStatementProvider deleteStatement); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + @InsertProvider(type=SqlProviderAdapter.class, method="insert") + int insert(InsertStatementProvider insertStatement); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + @InsertProvider(type=SqlProviderAdapter.class, method="insertMultiple") + int insertMultiple(MultiRowInsertStatementProvider multipleInsertStatement); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + @SelectProvider(type=SqlProviderAdapter.class, method="select") + @ResultMap("BookCommentReplyResult") + Optional selectOne(SelectStatementProvider selectStatement); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + @SelectProvider(type=SqlProviderAdapter.class, method="select") + @Results(id="BookCommentReplyResult", value = { + @Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true), + @Result(column="comment_id", property="commentId", jdbcType=JdbcType.BIGINT), + @Result(column="reply_content", property="replyContent", jdbcType=JdbcType.VARCHAR), + @Result(column="location", property="location", jdbcType=JdbcType.VARCHAR), + @Result(column="audit_status", property="auditStatus", jdbcType=JdbcType.TINYINT), + @Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP), + @Result(column="create_user_id", property="createUserId", jdbcType=JdbcType.BIGINT) + }) + List selectMany(SelectStatementProvider selectStatement); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + @UpdateProvider(type=SqlProviderAdapter.class, method="update") + int update(UpdateStatementProvider updateStatement); + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default long count(CountDSLCompleter completer) { + return MyBatis3Utils.countFrom(this::count, bookCommentReply, completer); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int delete(DeleteDSLCompleter completer) { + return MyBatis3Utils.deleteFrom(this::delete, bookCommentReply, completer); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int deleteByPrimaryKey(Long id_) { + return delete(c -> + c.where(id, isEqualTo(id_)) + ); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int insert(BookCommentReply record) { + return MyBatis3Utils.insert(this::insert, record, bookCommentReply, c -> + c.map(id).toProperty("id") + .map(commentId).toProperty("commentId") + .map(replyContent).toProperty("replyContent") + .map(location).toProperty("location") + .map(auditStatus).toProperty("auditStatus") + .map(createTime).toProperty("createTime") + .map(createUserId).toProperty("createUserId") + ); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int insertMultiple(Collection records) { + return MyBatis3Utils.insertMultiple(this::insertMultiple, records, bookCommentReply, c -> + c.map(id).toProperty("id") + .map(commentId).toProperty("commentId") + .map(replyContent).toProperty("replyContent") + .map(location).toProperty("location") + .map(auditStatus).toProperty("auditStatus") + .map(createTime).toProperty("createTime") + .map(createUserId).toProperty("createUserId") + ); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int insertSelective(BookCommentReply record) { + return MyBatis3Utils.insert(this::insert, record, bookCommentReply, c -> + c.map(id).toPropertyWhenPresent("id", record::getId) + .map(commentId).toPropertyWhenPresent("commentId", record::getCommentId) + .map(replyContent).toPropertyWhenPresent("replyContent", record::getReplyContent) + .map(location).toPropertyWhenPresent("location", record::getLocation) + .map(auditStatus).toPropertyWhenPresent("auditStatus", record::getAuditStatus) + .map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime) + .map(createUserId).toPropertyWhenPresent("createUserId", record::getCreateUserId) + ); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default Optional selectOne(SelectDSLCompleter completer) { + return MyBatis3Utils.selectOne(this::selectOne, selectList, bookCommentReply, completer); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default List select(SelectDSLCompleter completer) { + return MyBatis3Utils.selectList(this::selectMany, selectList, bookCommentReply, completer); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default List selectDistinct(SelectDSLCompleter completer) { + return MyBatis3Utils.selectDistinct(this::selectMany, selectList, bookCommentReply, completer); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default Optional selectByPrimaryKey(Long id_) { + return selectOne(c -> + c.where(id, isEqualTo(id_)) + ); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int update(UpdateDSLCompleter completer) { + return MyBatis3Utils.update(this::update, bookCommentReply, completer); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + static UpdateDSL updateAllColumns(BookCommentReply record, UpdateDSL dsl) { + return dsl.set(id).equalTo(record::getId) + .set(commentId).equalTo(record::getCommentId) + .set(replyContent).equalTo(record::getReplyContent) + .set(location).equalTo(record::getLocation) + .set(auditStatus).equalTo(record::getAuditStatus) + .set(createTime).equalTo(record::getCreateTime) + .set(createUserId).equalTo(record::getCreateUserId); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + static UpdateDSL updateSelectiveColumns(BookCommentReply record, UpdateDSL dsl) { + return dsl.set(id).equalToWhenPresent(record::getId) + .set(commentId).equalToWhenPresent(record::getCommentId) + .set(replyContent).equalToWhenPresent(record::getReplyContent) + .set(location).equalToWhenPresent(record::getLocation) + .set(auditStatus).equalToWhenPresent(record::getAuditStatus) + .set(createTime).equalToWhenPresent(record::getCreateTime) + .set(createUserId).equalToWhenPresent(record::getCreateUserId); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int updateByPrimaryKey(BookCommentReply record) { + return update(c -> + c.set(commentId).equalTo(record::getCommentId) + .set(replyContent).equalTo(record::getReplyContent) + .set(location).equalTo(record::getLocation) + .set(auditStatus).equalTo(record::getAuditStatus) + .set(createTime).equalTo(record::getCreateTime) + .set(createUserId).equalTo(record::getCreateUserId) + .where(id, isEqualTo(record::getId)) + ); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int updateByPrimaryKeySelective(BookCommentReply record) { + return update(c -> + c.set(commentId).equalToWhenPresent(record::getCommentId) + .set(replyContent).equalToWhenPresent(record::getReplyContent) + .set(location).equalToWhenPresent(record::getLocation) + .set(auditStatus).equalToWhenPresent(record::getAuditStatus) + .set(createTime).equalToWhenPresent(record::getCreateTime) + .set(createUserId).equalToWhenPresent(record::getCreateUserId) + .where(id, isEqualTo(record::getId)) + ); + } +} \ No newline at end of file diff --git a/novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskDynamicSqlSupport.java b/novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskDynamicSqlSupport.java index f26a068..fad7e2c 100644 --- a/novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskDynamicSqlSupport.java +++ b/novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskDynamicSqlSupport.java @@ -37,6 +37,9 @@ public final class CrawlSingleTaskDynamicSqlSupport { @Generated("org.mybatis.generator.api.MyBatisGenerator") public static final SqlColumn excCount = crawlSingleTask.excCount; + @Generated("org.mybatis.generator.api.MyBatisGenerator") + public static final SqlColumn crawlChapters = crawlSingleTask.crawlChapters; + @Generated("org.mybatis.generator.api.MyBatisGenerator") public static final SqlColumn createTime = crawlSingleTask.createTime; @@ -60,6 +63,8 @@ public final class CrawlSingleTaskDynamicSqlSupport { public final SqlColumn excCount = column("exc_count", JDBCType.TINYINT); + public final SqlColumn crawlChapters = column("crawl_chapters", JDBCType.INTEGER); + public final SqlColumn createTime = column("create_time", JDBCType.TIMESTAMP); public CrawlSingleTask() { diff --git a/novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskMapper.java b/novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskMapper.java index 646bc1e..5578448 100644 --- a/novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskMapper.java +++ b/novel-common/src/main/java/com/java2nb/novel/mapper/CrawlSingleTaskMapper.java @@ -35,7 +35,7 @@ import org.mybatis.dynamic.sql.util.mybatis3.MyBatis3Utils; @Mapper public interface CrawlSingleTaskMapper { @Generated("org.mybatis.generator.api.MyBatisGenerator") - BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceName, sourceBookId, catId, bookName, authorName, taskStatus, excCount, createTime); + BasicColumn[] selectList = BasicColumn.columnList(id, sourceId, sourceName, sourceBookId, catId, bookName, authorName, taskStatus, excCount, crawlChapters, createTime); @Generated("org.mybatis.generator.api.MyBatisGenerator") @SelectProvider(type=SqlProviderAdapter.class, method="select") @@ -70,6 +70,7 @@ public interface CrawlSingleTaskMapper { @Result(column="author_name", property="authorName", jdbcType=JdbcType.VARCHAR), @Result(column="task_status", property="taskStatus", jdbcType=JdbcType.TINYINT), @Result(column="exc_count", property="excCount", jdbcType=JdbcType.TINYINT), + @Result(column="crawl_chapters", property="crawlChapters", jdbcType=JdbcType.INTEGER), @Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP) }) List selectMany(SelectStatementProvider selectStatement); @@ -90,7 +91,7 @@ public interface CrawlSingleTaskMapper { @Generated("org.mybatis.generator.api.MyBatisGenerator") default int deleteByPrimaryKey(Long id_) { - return delete(c -> + return delete(c -> c.where(id, isEqualTo(id_)) ); } @@ -99,15 +100,16 @@ public interface CrawlSingleTaskMapper { default int insert(CrawlSingleTask record) { return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c -> c.map(id).toProperty("id") - .map(sourceId).toProperty("sourceId") - .map(sourceName).toProperty("sourceName") - .map(sourceBookId).toProperty("sourceBookId") - .map(catId).toProperty("catId") - .map(bookName).toProperty("bookName") - .map(authorName).toProperty("authorName") - .map(taskStatus).toProperty("taskStatus") - .map(excCount).toProperty("excCount") - .map(createTime).toProperty("createTime") + .map(sourceId).toProperty("sourceId") + .map(sourceName).toProperty("sourceName") + .map(sourceBookId).toProperty("sourceBookId") + .map(catId).toProperty("catId") + .map(bookName).toProperty("bookName") + .map(authorName).toProperty("authorName") + .map(taskStatus).toProperty("taskStatus") + .map(excCount).toProperty("excCount") + .map(crawlChapters).toProperty("crawlChapters") + .map(createTime).toProperty("createTime") ); } @@ -115,15 +117,16 @@ public interface CrawlSingleTaskMapper { default int insertMultiple(Collection records) { return MyBatis3Utils.insertMultiple(this::insertMultiple, records, crawlSingleTask, c -> c.map(id).toProperty("id") - .map(sourceId).toProperty("sourceId") - .map(sourceName).toProperty("sourceName") - .map(sourceBookId).toProperty("sourceBookId") - .map(catId).toProperty("catId") - .map(bookName).toProperty("bookName") - .map(authorName).toProperty("authorName") - .map(taskStatus).toProperty("taskStatus") - .map(excCount).toProperty("excCount") - .map(createTime).toProperty("createTime") + .map(sourceId).toProperty("sourceId") + .map(sourceName).toProperty("sourceName") + .map(sourceBookId).toProperty("sourceBookId") + .map(catId).toProperty("catId") + .map(bookName).toProperty("bookName") + .map(authorName).toProperty("authorName") + .map(taskStatus).toProperty("taskStatus") + .map(excCount).toProperty("excCount") + .map(crawlChapters).toProperty("crawlChapters") + .map(createTime).toProperty("createTime") ); } @@ -131,15 +134,16 @@ public interface CrawlSingleTaskMapper { default int insertSelective(CrawlSingleTask record) { return MyBatis3Utils.insert(this::insert, record, crawlSingleTask, c -> c.map(id).toPropertyWhenPresent("id", record::getId) - .map(sourceId).toPropertyWhenPresent("sourceId", record::getSourceId) - .map(sourceName).toPropertyWhenPresent("sourceName", record::getSourceName) - .map(sourceBookId).toPropertyWhenPresent("sourceBookId", record::getSourceBookId) - .map(catId).toPropertyWhenPresent("catId", record::getCatId) - .map(bookName).toPropertyWhenPresent("bookName", record::getBookName) - .map(authorName).toPropertyWhenPresent("authorName", record::getAuthorName) - .map(taskStatus).toPropertyWhenPresent("taskStatus", record::getTaskStatus) - .map(excCount).toPropertyWhenPresent("excCount", record::getExcCount) - .map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime) + .map(sourceId).toPropertyWhenPresent("sourceId", record::getSourceId) + .map(sourceName).toPropertyWhenPresent("sourceName", record::getSourceName) + .map(sourceBookId).toPropertyWhenPresent("sourceBookId", record::getSourceBookId) + .map(catId).toPropertyWhenPresent("catId", record::getCatId) + .map(bookName).toPropertyWhenPresent("bookName", record::getBookName) + .map(authorName).toPropertyWhenPresent("authorName", record::getAuthorName) + .map(taskStatus).toPropertyWhenPresent("taskStatus", record::getTaskStatus) + .map(excCount).toPropertyWhenPresent("excCount", record::getExcCount) + .map(crawlChapters).toPropertyWhenPresent("crawlChapters", record::getCrawlChapters) + .map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime) ); } @@ -173,35 +177,7 @@ public interface CrawlSingleTaskMapper { @Generated("org.mybatis.generator.api.MyBatisGenerator") static UpdateDSL updateAllColumns(CrawlSingleTask record, UpdateDSL dsl) { return dsl.set(id).equalTo(record::getId) - .set(sourceId).equalTo(record::getSourceId) - .set(sourceName).equalTo(record::getSourceName) - .set(sourceBookId).equalTo(record::getSourceBookId) - .set(catId).equalTo(record::getCatId) - .set(bookName).equalTo(record::getBookName) - .set(authorName).equalTo(record::getAuthorName) - .set(taskStatus).equalTo(record::getTaskStatus) - .set(excCount).equalTo(record::getExcCount) - .set(createTime).equalTo(record::getCreateTime); - } - - @Generated("org.mybatis.generator.api.MyBatisGenerator") - static UpdateDSL updateSelectiveColumns(CrawlSingleTask record, UpdateDSL dsl) { - return dsl.set(id).equalToWhenPresent(record::getId) - .set(sourceId).equalToWhenPresent(record::getSourceId) - .set(sourceName).equalToWhenPresent(record::getSourceName) - .set(sourceBookId).equalToWhenPresent(record::getSourceBookId) - .set(catId).equalToWhenPresent(record::getCatId) - .set(bookName).equalToWhenPresent(record::getBookName) - .set(authorName).equalToWhenPresent(record::getAuthorName) - .set(taskStatus).equalToWhenPresent(record::getTaskStatus) - .set(excCount).equalToWhenPresent(record::getExcCount) - .set(createTime).equalToWhenPresent(record::getCreateTime); - } - - @Generated("org.mybatis.generator.api.MyBatisGenerator") - default int updateByPrimaryKey(CrawlSingleTask record) { - return update(c -> - c.set(sourceId).equalTo(record::getSourceId) + .set(sourceId).equalTo(record::getSourceId) .set(sourceName).equalTo(record::getSourceName) .set(sourceBookId).equalTo(record::getSourceBookId) .set(catId).equalTo(record::getCatId) @@ -209,15 +185,14 @@ public interface CrawlSingleTaskMapper { .set(authorName).equalTo(record::getAuthorName) .set(taskStatus).equalTo(record::getTaskStatus) .set(excCount).equalTo(record::getExcCount) - .set(createTime).equalTo(record::getCreateTime) - .where(id, isEqualTo(record::getId)) - ); + .set(crawlChapters).equalTo(record::getCrawlChapters) + .set(createTime).equalTo(record::getCreateTime); } @Generated("org.mybatis.generator.api.MyBatisGenerator") - default int updateByPrimaryKeySelective(CrawlSingleTask record) { - return update(c -> - c.set(sourceId).equalToWhenPresent(record::getSourceId) + static UpdateDSL updateSelectiveColumns(CrawlSingleTask record, UpdateDSL dsl) { + return dsl.set(id).equalToWhenPresent(record::getId) + .set(sourceId).equalToWhenPresent(record::getSourceId) .set(sourceName).equalToWhenPresent(record::getSourceName) .set(sourceBookId).equalToWhenPresent(record::getSourceBookId) .set(catId).equalToWhenPresent(record::getCatId) @@ -225,8 +200,41 @@ public interface CrawlSingleTaskMapper { .set(authorName).equalToWhenPresent(record::getAuthorName) .set(taskStatus).equalToWhenPresent(record::getTaskStatus) .set(excCount).equalToWhenPresent(record::getExcCount) - .set(createTime).equalToWhenPresent(record::getCreateTime) - .where(id, isEqualTo(record::getId)) + .set(crawlChapters).equalToWhenPresent(record::getCrawlChapters) + .set(createTime).equalToWhenPresent(record::getCreateTime); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int updateByPrimaryKey(CrawlSingleTask record) { + return update(c -> + c.set(sourceId).equalTo(record::getSourceId) + .set(sourceName).equalTo(record::getSourceName) + .set(sourceBookId).equalTo(record::getSourceBookId) + .set(catId).equalTo(record::getCatId) + .set(bookName).equalTo(record::getBookName) + .set(authorName).equalTo(record::getAuthorName) + .set(taskStatus).equalTo(record::getTaskStatus) + .set(excCount).equalTo(record::getExcCount) + .set(crawlChapters).equalTo(record::getCrawlChapters) + .set(createTime).equalTo(record::getCreateTime) + .where(id, isEqualTo(record::getId)) + ); + } + + @Generated("org.mybatis.generator.api.MyBatisGenerator") + default int updateByPrimaryKeySelective(CrawlSingleTask record) { + return update(c -> + c.set(sourceId).equalToWhenPresent(record::getSourceId) + .set(sourceName).equalToWhenPresent(record::getSourceName) + .set(sourceBookId).equalToWhenPresent(record::getSourceBookId) + .set(catId).equalToWhenPresent(record::getCatId) + .set(bookName).equalToWhenPresent(record::getBookName) + .set(authorName).equalToWhenPresent(record::getAuthorName) + .set(taskStatus).equalToWhenPresent(record::getTaskStatus) + .set(excCount).equalToWhenPresent(record::getExcCount) + .set(crawlChapters).equalToWhenPresent(record::getCrawlChapters) + .set(createTime).equalToWhenPresent(record::getCreateTime) + .where(id, isEqualTo(record::getId)) ); } } \ No newline at end of file diff --git a/novel-common/src/main/resources/application-common-dev.yml b/novel-common/src/main/resources/application-common-dev.yml index a475996..003144c 100644 --- a/novel-common/src/main/resources/application-common-dev.yml +++ b/novel-common/src/main/resources/application-common-dev.yml @@ -1,71 +1,18 @@ spring: - profiles: - include: [ common ] + config: + import: classpath:application-common.yml main: allow-bean-definition-overriding: true #Redis服务器IP - redis: - host: 127.0.0.1 - #Redis服务器连接端口 - port: 6379 - #Redis服务器连接密码 - password: test123456 - jedis: - pool: - #连接池最大连接数(使用负值表示没有限制) - max-active: 8 - #连接池最大阻塞等待时间(使用负值表示没有限制) - max-wait: 1 - #连接池最大阻塞等待时间(使用负值表示没有限制) - max-idle: 8 - #连接池中的最小空闲连接 - min-idle: 0 - #连接超时时间(毫秒) - timeout: 30000 - datasource: - url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - driver-class-name: com.mysql.cj.jdbc.Driver - - -####使用shardingJdbc时, -####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误 -##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR - -sharding: - jdbc: - datasource: - names: ds0 #,ds1 - ds0: - type: com.zaxxer.hikari.HikariDataSource - driver-class-name: com.mysql.cj.jdbc.Driver - jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - # ds1: - # type: com.alibaba.druid.pool.DruidDataSource - # driver-class-name: com.mysql.jdbc.Driver - # url: jdbc:mysql://localhost:3306/novel_plus2 - # username: root - # password: test123456 - config: - sharding: - props: - sql.show: true - tables: - book_content: #book_content表 - key-generator-column-name: id #主键 - actual-data-nodes: ds${0}.book_content${0..9} #数据节点 - # database-strategy: #分库策略 - # inline: - # sharding-column: book_id - # algorithm-expression: ds${book_id % 10} - table-strategy: #分表策略 - inline: - shardingColumn: index_id - algorithm-expression: book_content${index_id % 10} - + data: + redis: + host: 127.0.0.1 + #Redis服务器连接端口 + port: 6379 + #Redis服务器连接密码 + password: test123456 + #连接超时时间(毫秒) + timeout: 10000 content: save: @@ -78,6 +25,10 @@ http: # 是否开启 HTTP 代理,true-开启,false-不开启 enabled: false # 代理 IP - ip: u493.kdltps.com + ip: us.swiftproxy.net # 代理端口号 - port: 15818 \ No newline at end of file + port: 7878 + # 代理用户名 + username: swiftproxy_u + # 代理密码 + password: swiftproxy_p \ No newline at end of file diff --git a/novel-common/src/main/resources/application-common-prod.yml b/novel-common/src/main/resources/application-common-prod.yml index df2b262..3afeae2 100644 --- a/novel-common/src/main/resources/application-common-prod.yml +++ b/novel-common/src/main/resources/application-common-prod.yml @@ -1,71 +1,18 @@ spring: - profiles: - include: [ common ] + config: + import: classpath:application-common.yml main: allow-bean-definition-overriding: true - #Redis服务器IP - redis: - host: 127.0.0.1 - #Redis服务器连接端口 - port: 6379 - #Redis服务器连接密码 - password: test - jedis: - pool: - #连接池最大连接数(使用负值表示没有限制) - max-active: 8 - #连接池最大阻塞等待时间(使用负值表示没有限制) - max-wait: 1 - #连接池最大阻塞等待时间(使用负值表示没有限制) - max-idle: 8 - #连接池中的最小空闲连接 - min-idle: 0 - #连接超时时间(毫秒) - timeout: 30000 - datasource: - url: jdbc:mysql://127.0.0.1:3306/novel_biz?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - driver-class-name: com.mysql.cj.jdbc.Driver - - -####使用shardingJdbc时, -####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误 -##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR - -sharding: - jdbc: - datasource: - names: ds0 #,ds1 - ds0: - type: com.zaxxer.hikari.HikariDataSource - driver-class-name: com.mysql.cj.jdbc.Driver - jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - # ds1: - # type: com.alibaba.druid.pool.DruidDataSource - # driver-class-name: com.mysql.jdbc.Driver - # url: jdbc:mysql://localhost:3306/novel_plus2 - # username: root - # password: test123456 - config: - sharding: - props: - sql.show: true - tables: - book_content: #book_content表 - key-generator-column-name: id #主键 - actual-data-nodes: ds${0}.book_content${0..9} #数据节点 - # database-strategy: #分库策略 - # inline: - # sharding-column: book_id - # algorithm-expression: ds${book_id % 10} - table-strategy: #分表策略 - inline: - shardingColumn: index_id - algorithm-expression: book_content${index_id % 10} - + data: + redis: + #Redis服务器IP + host: 127.0.0.1 + #Redis服务器连接端口 + port: 6379 + #Redis服务器连接密码 + password: test123456 + #连接超时时间(毫秒) + timeout: 10000 logging: level: diff --git a/novel-common/src/main/resources/application-common.yml b/novel-common/src/main/resources/application-common.yml index 08d4b46..6bc63b2 100644 --- a/novel-common/src/main/resources/application-common.yml +++ b/novel-common/src/main/resources/application-common.yml @@ -1,16 +1,11 @@ spring: - cache: - ehcache: - config: classpath:ehcache.xml + datasource: + url: jdbc:shardingsphere:absolutepath:${user.dir}/config/shardingsphere-jdbc.yml + driver-class-name: org.apache.shardingsphere.driver.ShardingSphereDriver thymeleaf: mode: LEGACYHTML5 #去除thymeleaf的html严格校验thymeleaf.mode=LEGACYHTML5 cache: false # 是否开启模板缓存,默认true,建议在开发时关闭缓存,不然没法看到实时 - # 将所有数字转为 String 类型返回,避免前端数据精度丢失的问题 - jackson: - generator: - write-numbers-as-strings: true - #上传文件的最大值(100M) servlet: multipart: @@ -27,6 +22,8 @@ mybatis: logging: config: classpath:logback-boot.xml +pagehelper: + helper-dialect: mysql diff --git a/novel-crawl/pom.xml b/novel-crawl/pom.xml index a3958f9..4a9682c 100644 --- a/novel-crawl/pom.xml +++ b/novel-crawl/pom.xml @@ -5,7 +5,7 @@ novel com.java2nb - 4.3.0 + 5.2.0 4.0.0 @@ -21,6 +21,12 @@ novel-common + + org.yaml + snakeyaml + 2.2 + + org.springframework.boot spring-boot-starter-security @@ -29,7 +35,6 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.version} diff --git a/novel-crawl/src/main/build/config/application-common-prod.yml b/novel-crawl/src/main/build/config/application-common-prod.yml deleted file mode 100644 index 86a6abd..0000000 --- a/novel-crawl/src/main/build/config/application-common-prod.yml +++ /dev/null @@ -1,53 +0,0 @@ -#端口号 -server: - port: 8083 - -#不分表的数据库配置 -spring: - datasource: - url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: root - redis: - host: 127.0.0.1 - port: 6379 - password: test123456 - -####使用shardingJdbc时, -####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误 -##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR -sharding: - jdbc: - datasource: - ds0: - jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai - username: root - password: test123456 - -#登录用户名密码 -admin: - username: admin - password: admin -# -##爬虫自动更新的线程数 -##建议小说数量不多或者正在运行新书入库爬虫的情况下设置为1即可 -##随着小说数量的增多可以逐渐增加,但建议不要超出CPU的线程数 -crawl: - update: - thread: 1 - -#小说内容保存配置 -content: - save: - storage: db # 小说内容存储方式:db-数据库,txt-TXT文本 - path: /Users/xiongxiaoyang/books # 小说TXT文本保存路径 - -# HTTP 代理配置 -http: - proxy: - # 是否开启 HTTP 代理,true-开启,false-不开启 - enabled: false - # 代理 IP - ip: u493.kdltps.com - # 代理端口号 - port: 15818 \ No newline at end of file diff --git a/novel-crawl/src/main/build/config/application.yml b/novel-crawl/src/main/build/config/application.yml new file mode 100644 index 0000000..370d2b4 --- /dev/null +++ b/novel-crawl/src/main/build/config/application.yml @@ -0,0 +1,47 @@ +#端口号 +server: + port: 8083 + +spring: + data: + redis: + #Redis服务器IP + host: 127.0.0.1 + #Redis服务器连接端口 + port: 6379 + #Redis服务器连接密码 + password: test123456 + #连接超时时间(毫秒) + timeout: 10000 + +#登录用户名密码 +admin: + username: admin + password: admin +# +##爬虫自动更新的线程数 +##建议小说数量不多或者正在运行新书入库爬虫的情况下设置为1即可 +##随着小说数量的增多可以逐渐增加,但建议不要超出CPU的线程数 +crawl: + update: + thread: 1 + +#小说内容保存配置 +content: + save: + storage: db # 小说内容存储方式:db-数据库,txt-TXT文本 + path: /Users/xiongxiaoyang/books # 小说TXT文本保存路径 + +# HTTP 代理配置 +http: + proxy: + # 是否开启 HTTP 代理,true-开启,false-不开启 + enabled: false + # 代理 IP + ip: us.swiftproxy.net + # 代理端口号 + port: 7878 + # 代理用户名 + username: swiftproxy_u + # 代理密码 + password: swiftproxy_p \ No newline at end of file diff --git a/novel-crawl/src/main/build/config/shardingsphere-jdbc.yml b/novel-crawl/src/main/build/config/shardingsphere-jdbc.yml new file mode 100644 index 0000000..8659c28 --- /dev/null +++ b/novel-crawl/src/main/build/config/shardingsphere-jdbc.yml @@ -0,0 +1,47 @@ +mode: + # 单机模式 + type: Standalone + # 元数据持久化 + repository: + # 数据库持久化 + type: JDBC + +# 数据源配置 +dataSources: + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.mysql.cj.jdbc.Driver + jdbcUrl: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai + username: root + password: test123456 +# 规则配置 +rules: + - !SINGLE + tables: + - "*.*" + - !SHARDING + tables: # 数据分片规则配置 + book_content: + # 分库策略,缺省表示使用默认分库策略 + actualDataNodes: ds_${1}.book_content${0..9} + # 分表策略 + tableStrategy: + standard: + # 分片列名称 + shardingColumn: index_id + # 分片算法名称 + shardingAlgorithmName: bookContentSharding + + shardingAlgorithms: + bookContentSharding: + # 行表达式分片算法,使用 Groovy 的表达式,提供对 SQL 语句中的 = 和 IN 的分片操作支持 + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: book_content${index_id % 10} + + + +props: + # 是否在日志中打印 SQL + sql-show: true diff --git a/novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java b/novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java index 6b312fc..84f8de0 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java @@ -153,6 +153,14 @@ public class CrawlController { return RestResult.ok(); } + /** + * 采集任务进度查询 + * */ + @GetMapping("getTaskProgress/{id}") + public RestResult getTaskProgress(@PathVariable("id") Long id){ + return RestResult.ok(crawlService.getTaskProgress(id)); + } + diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/config/SecurityConfiguration.java b/novel-crawl/src/main/java/com/java2nb/novel/core/config/SecurityConfiguration.java index ff35ebc..f684fe4 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/config/SecurityConfiguration.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/core/config/SecurityConfiguration.java @@ -4,14 +4,15 @@ import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; +import org.springframework.security.config.Customizer; import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.core.userdetails.User; +import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.security.provisioning.InMemoryUserDetailsManager; +import org.springframework.security.web.SecurityFilterChain; /** * SpringSecurity配置 @@ -21,7 +22,7 @@ import org.springframework.security.crypto.password.PasswordEncoder; @Configuration @EnableWebSecurity @RequiredArgsConstructor -public class SecurityConfiguration extends WebSecurityConfigurerAdapter { +public class SecurityConfiguration { @Value("${admin.username}") private String username; @@ -29,39 +30,40 @@ public class SecurityConfiguration extends WebSecurityConfigurerAdapter { @Value("${admin.password}") private String password; - @Bean public PasswordEncoder passwordEncoder() { return new BCryptPasswordEncoder(); } - - @Override - public void configure(WebSecurity web) throws Exception { - super.configure(web); + @Bean + public InMemoryUserDetailsManager userDetailsService() { + UserDetails admin = User.builder() + .username(username) + .password(passwordEncoder().encode(password)) + .roles("ADMIN") + .build(); + return new InMemoryUserDetailsManager(admin); } - @Override - public void configure(AuthenticationManagerBuilder auth) throws Exception { + @Bean + public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception { + http + .csrf(csrf -> csrf.disable()) // 禁用 CSRF + .authorizeHttpRequests(auth -> auth + .requestMatchers("/css/**", "/favicon.ico").permitAll() // 允许访问静态资源 + .anyRequest().hasRole("ADMIN") // 其他请求需要 ADMIN 角色 + ) + .formLogin(form -> form + .loginPage("/login.html") // 自定义登录页面 + .loginProcessingUrl("/login") // 登录处理 URL + .permitAll() + ) + .logout(logout -> logout + .logoutUrl("/logout") // 登出 URL + .logoutSuccessUrl("/") // 登出成功后跳转的页面 + ) + .httpBasic(Customizer.withDefaults()); // 启用 HTTP Basic 认证 - User.UserBuilder builder = User.builder().passwordEncoder(passwordEncoder()::encode); - auth.inMemoryAuthentication().withUser(builder.username(username).password(password).roles("ADMIN").build()); + return http.build(); } - - @Override - protected void configure(HttpSecurity http) throws Exception { - http.csrf().disable() - .authorizeRequests() - .antMatchers("/css/**").permitAll() - .antMatchers("/favicon.ico").permitAll() - .antMatchers("/**").hasRole("ADMIN") - .and().formLogin().loginPage("/login.html").loginProcessingUrl("/login").permitAll() - .and().logout() - .logoutUrl("/logout") - .logoutSuccessUrl("/") - .and().httpBasic(); - - } - - -} +} \ No newline at end of file diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlBookHandler.java b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlBookHandler.java index 7d6a8c0..9fdab12 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlBookHandler.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlBookHandler.java @@ -7,6 +7,6 @@ import com.java2nb.novel.entity.Book; * */ public interface CrawlBookHandler { - void handle(Book book); + void handle(Book book) throws InterruptedException; } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java index 192c4e4..a08fd3d 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/CrawlParser.java @@ -1,21 +1,20 @@ package com.java2nb.novel.core.crawl; -import com.java2nb.novel.core.utils.HttpUtil; import com.java2nb.novel.core.utils.RandomBookInfoUtil; -import com.java2nb.novel.core.utils.RestTemplateUtil; import com.java2nb.novel.core.utils.StringUtil; import com.java2nb.novel.entity.Book; import com.java2nb.novel.entity.BookContent; import com.java2nb.novel.entity.BookIndex; +import com.java2nb.novel.entity.CrawlSingleTask; import com.java2nb.novel.utils.Constants; +import com.java2nb.novel.utils.CrawlHttpClient; import io.github.xxyopen.util.IdWorker; -import lombok.SneakyThrows; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.RestTemplate; +import org.springframework.stereotype.Component; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; import java.util.regex.Matcher; @@ -27,19 +26,38 @@ import java.util.regex.Pattern; * @author Administrator */ @Slf4j +@Component +@RequiredArgsConstructor public class CrawlParser { - private static final IdWorker idWorker = IdWorker.INSTANCE; + private final IdWorker ID_WORKER = IdWorker.INSTANCE; - private static final RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8"); + private final CrawlHttpClient crawlHttpClient; - private static final ThreadLocal retryCount = new ThreadLocal<>(); + /** + * 爬虫任务进度 + */ + private final Map crawlTaskProgress = new HashMap<>(); - @SneakyThrows - public static void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler) { + /** + * 获取爬虫任务进度 + */ + public Integer getCrawlTaskProgress(Long taskId) { + return crawlTaskProgress.get(taskId); + } + + /** + * 移除爬虫任务进度 + */ + public void removeCrawlTaskProgress(Long taskId) { + crawlTaskProgress.remove(taskId); + } + + public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler) + throws InterruptedException { Book book = new Book(); String bookDetailUrl = ruleBean.getBookDetailUrl().replace("{bookId}", bookId); - String bookDetailHtml = getByHttpClientWithChrome(bookDetailUrl); + String bookDetailHtml = crawlHttpClient.get(bookDetailUrl, ruleBean.getCharset()); if (bookDetailHtml != null) { Pattern bookNamePatten = PatternFactory.getPattern(ruleBean.getBookNamePatten()); Matcher bookNameMatch = bookNamePatten.matcher(bookDetailHtml); @@ -89,14 +107,31 @@ public class CrawlParser { } } - String desc = bookDetailHtml.substring(bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length()); + String desc = bookDetailHtml.substring( + bookDetailHtml.indexOf(ruleBean.getDescStart()) + ruleBean.getDescStart().length()); desc = desc.substring(0, desc.indexOf(ruleBean.getDescEnd())); //过滤掉简介中的特殊标签 desc = desc.replaceAll("", "") - .replaceAll("", "") - .replaceAll("

\\s*

", "") - .replaceAll("

", "") - .replaceAll("

", "
"); + .replaceAll("", "") + .replaceAll("

\\s*

", "") + .replaceAll("

", "") + .replaceAll("

", "
"); + // 小说简介过滤 + String filterDesc = ruleBean.getFilterDesc(); + if (StringUtils.isNotBlank(filterDesc)) { + String[] filterRules = filterDesc.replace("\r\n", "\n").split("\n"); + for (String filterRule : filterRules) { + if (StringUtils.isNotBlank(filterRule)) { + desc = desc.replaceAll(filterRule, ""); + } + } + } + // 去除小说简介前后空格 + desc = desc.trim(); + // 去除小说简介末尾冗余的小说名 + if (desc.endsWith(bookName)) { + desc = desc.substring(0, desc.length() - bookName.length()); + } //设置书籍简介 book.setBookDesc(desc); if (StringUtils.isNotBlank(ruleBean.getStatusPatten())) { @@ -112,14 +147,20 @@ public class CrawlParser { } } - if (StringUtils.isNotBlank(ruleBean.getUpadateTimePatten()) && StringUtils.isNotBlank(ruleBean.getUpadateTimeFormatPatten())) { + if (StringUtils.isNotBlank(ruleBean.getUpadateTimePatten()) && StringUtils.isNotBlank( + ruleBean.getUpadateTimeFormatPatten())) { Pattern updateTimePatten = PatternFactory.getPattern(ruleBean.getUpadateTimePatten()); Matcher updateTimeMatch = updateTimePatten.matcher(bookDetailHtml); boolean isFindUpdateTime = updateTimeMatch.find(); if (isFindUpdateTime) { String updateTime = updateTimeMatch.group(1); //设置更新时间 - book.setLastIndexUpdateTime(new SimpleDateFormat(ruleBean.getUpadateTimeFormatPatten()).parse(updateTime)); + try { + book.setLastIndexUpdateTime( + new SimpleDateFormat(ruleBean.getUpadateTimeFormatPatten()).parse(updateTime)); + } catch (ParseException e) { + log.error("解析最新章节更新时间出错", e); + } } } @@ -131,7 +172,7 @@ public class CrawlParser { } else if (book.getVisitCount() != null && book.getScore() == null) { //随机根据访问次数生成评分 book.setScore(RandomBookInfoUtil.getScoreByVisitCount(book.getVisitCount())); - } else if (book.getVisitCount() == null && book.getScore() == null) { + } else if (book.getVisitCount() == null) { //都没有,设置成固定值 book.setVisitCount(Constants.VISIT_COUNT_DEFAULT); book.setScore(6.5f); @@ -141,7 +182,14 @@ public class CrawlParser { handler.handle(book); } - public static boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean, Map existBookIndexMap, CrawlBookChapterHandler handler) { + public boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean, + Map existBookIndexMap, CrawlBookChapterHandler handler, CrawlSingleTask task) + throws InterruptedException { + + if (task != null) { + // 开始采集 + crawlTaskProgress.put(task.getId(), 0); + } Date currentDate = new Date(); @@ -149,11 +197,12 @@ public class CrawlParser { List contentList = new ArrayList<>(); //读取目录 String indexListUrl = ruleBean.getBookIndexUrl().replace("{bookId}", sourceBookId); - String indexListHtml = getByHttpClientWithChrome(indexListUrl); + String indexListHtml = crawlHttpClient.get(indexListUrl, ruleBean.getCharset()); if (indexListHtml != null) { if (StringUtils.isNotBlank(ruleBean.getBookIndexStart())) { - indexListHtml = indexListHtml.substring(indexListHtml.indexOf(ruleBean.getBookIndexStart()) + ruleBean.getBookIndexStart().length()); + indexListHtml = indexListHtml.substring( + indexListHtml.indexOf(ruleBean.getBookIndexStart()) + ruleBean.getBookIndexStart().length()); } Pattern indexIdPatten = PatternFactory.getPattern(ruleBean.getIndexIdPatten()); @@ -174,14 +223,16 @@ public class CrawlParser { BookIndex hasIndex = existBookIndexMap.get(indexNum); String indexName = indexNameMatch.group(1); - if (hasIndex == null || !StringUtils.deleteWhitespace(hasIndex.getIndexName()).equals(StringUtils.deleteWhitespace(indexName))) { + if (hasIndex == null || !StringUtils.deleteWhitespace(hasIndex.getIndexName()) + .equals(StringUtils.deleteWhitespace(indexName))) { String sourceIndexId = indexIdMatch.group(1); String bookContentUrl = ruleBean.getBookContentUrl(); int calStart = bookContentUrl.indexOf("{cal_"); if (calStart != -1) { //内容页URL需要进行计算才能得到 - String calStr = bookContentUrl.substring(calStart, calStart + bookContentUrl.substring(calStart).indexOf("}")); + String calStr = bookContentUrl.substring(calStart, + calStart + bookContentUrl.substring(calStart).indexOf("}")); String[] calArr = calStr.split("_"); int calType = Integer.parseInt(calArr[1]); if (calType == 1) { @@ -196,7 +247,7 @@ public class CrawlParser { calResult = sourceIndexId.substring(0, sourceBookId.length() - y); } - if (calResult.length() == 0) { + if (calResult.isEmpty()) { calResult = "0"; } @@ -206,13 +257,27 @@ public class CrawlParser { } - String contentUrl = bookContentUrl.replace("{bookId}", sourceBookId).replace("{indexId}", sourceIndexId); + String contentUrl = bookContentUrl.replace("{bookId}", sourceBookId) + .replace("{indexId}", sourceIndexId); //查询章节内容 - String contentHtml = getByHttpClientWithChrome(contentUrl); + String contentHtml = crawlHttpClient.get(contentUrl, ruleBean.getCharset()); if (contentHtml != null && !contentHtml.contains("正在手打中")) { - String content = contentHtml.substring(contentHtml.indexOf(ruleBean.getContentStart()) + ruleBean.getContentStart().length()); + String content = contentHtml.substring( + contentHtml.indexOf(ruleBean.getContentStart()) + ruleBean.getContentStart().length()); content = content.substring(0, content.indexOf(ruleBean.getContentEnd())); + // 小说内容过滤 + String filterContent = ruleBean.getFilterContent(); + if (StringUtils.isNotBlank(filterContent)) { + String[] filterRules = filterContent.replace("\r\n", "\n").split("\n"); + for (String filterRule : filterRules) { + if (StringUtils.isNotBlank(filterRule)) { + content = content.replaceAll(filterRule, ""); + } + } + } + // 去除小说内容末尾的所有换行 + content = removeTrailingBrTags(content); //插入章节目录和章节内容 BookIndex bookIndex = new BookIndex(); bookIndex.setIndexName(indexName); @@ -235,7 +300,7 @@ public class CrawlParser { } else { //章节插入 //设置目录和章节内容 - Long indexId = idWorker.nextId(); + Long indexId = ID_WORKER.nextId(); bookIndex.setId(indexId); bookIndex.setBookId(book.getId()); @@ -248,6 +313,11 @@ public class CrawlParser { } bookIndex.setUpdateTime(currentDate); + if (task != null) { + // 更新采集进度 + crawlTaskProgress.put(task.getId(), indexList.size()); + } + } @@ -257,11 +327,10 @@ public class CrawlParser { isFindIndex = indexIdMatch.find() & indexNameMatch.find(); } - - if (indexList.size() > 0) { + if (!indexList.isEmpty()) { //如果有爬到最新章节,则设置小说主表的最新章节信息 //获取爬取到的最新章节 - BookIndex lastIndex = indexList.get(indexList.size() - 1); + BookIndex lastIndex = indexList.getLast(); book.setLastIndexId(lastIndex.getId()); book.setLastIndexName(lastIndex.getIndexName()); book.setLastIndexUpdateTime(currentDate); @@ -270,7 +339,7 @@ public class CrawlParser { book.setWordCount(totalWordCount); book.setUpdateTime(currentDate); - if (indexList.size() == contentList.size() && indexList.size() > 0) { + if (indexList.size() == contentList.size() && !indexList.isEmpty()) { handler.handle(new ChapterBean() {{ setBookIndexList(indexList); @@ -291,55 +360,11 @@ public class CrawlParser { } - - private static String getByHttpClient(String url) { - try { - ResponseEntity forEntity = restTemplate.getForEntity(url, String.class); - if (forEntity.getStatusCode() == HttpStatus.OK) { - String body = forEntity.getBody(); - assert body != null; - if (body.length() < Constants.INVALID_HTML_LENGTH) { - return processErrorHttpResult(url); - } - //成功获得html内容 - return body; - } - } catch (Exception e) { - e.printStackTrace(); - } - return processErrorHttpResult(url); - + /** + * 删除字符串末尾的所有
类似标签(允许各种空格) + */ + public static String removeTrailingBrTags(String str) { + return str.replaceAll("(?i)(?:\\s*<\\s*br\\s*/?\\s*>)++(?:\\s|\\u3000)*$", ""); } - private static String getByHttpClientWithChrome(String url) { - try { - - String body = HttpUtil.getByHttpClientWithChrome(url); - if (body != null && body.length() < Constants.INVALID_HTML_LENGTH) { - return processErrorHttpResult(url); - } - //成功获得html内容 - return body; - } catch (Exception e) { - e.printStackTrace(); - } - return processErrorHttpResult(url); - - } - - @SneakyThrows - private static String processErrorHttpResult(String url) { - Integer count = retryCount.get(); - if (count == null) { - count = 0; - } - if (count < Constants.HTTP_FAIL_RETRY_COUNT) { - Thread.sleep(new Random().nextInt(10 * 1000)); - retryCount.set(++count); - return getByHttpClient(url); - } - return null; - } - - } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/RuleBean.java b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/RuleBean.java index 6b42685..205d785 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/RuleBean.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/core/crawl/RuleBean.java @@ -1,29 +1,37 @@ package com.java2nb.novel.core.crawl; +import com.java2nb.novel.utils.Constants; import lombok.Data; import java.util.Map; /** * 爬虫解析规则bean + * * @author Administrator */ @Data public class RuleBean { + /** + * 网页字符编码 + */ + private String charset = Constants.CRAWL_DEFAULT_CHARSET; + + /** * 小说更新列表url - * */ + */ private String updateBookListUrl; /** * 分类列表页URL规则 - * */ + */ private String bookListUrl; - private Map catIdRule; + private Map catIdRule; - private Map bookStatusRule; + private Map bookStatusRule; private String bookIdPatten; private String pagePatten; @@ -37,6 +45,7 @@ public class RuleBean { private String visitCountPatten; private String descStart; private String descEnd; + private String filterDesc; private String upadateTimePatten; private String upadateTimeFormatPatten; private String bookIndexUrl; @@ -51,5 +60,7 @@ public class RuleBean { private String bookIndexStart; + private String filterContent; + } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java b/novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java index ad57357..4ca21cd 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java @@ -1,21 +1,23 @@ package com.java2nb.novel.core.listener; import com.fasterxml.jackson.databind.ObjectMapper; -import com.java2nb.novel.core.crawl.ChapterBean; import com.java2nb.novel.core.crawl.CrawlParser; import com.java2nb.novel.core.crawl.RuleBean; -import com.java2nb.novel.entity.*; +import com.java2nb.novel.entity.Book; +import com.java2nb.novel.entity.BookIndex; +import com.java2nb.novel.entity.CrawlSingleTask; +import com.java2nb.novel.entity.CrawlSource; import com.java2nb.novel.service.BookService; import com.java2nb.novel.service.CrawlService; import com.java2nb.novel.utils.Constants; +import jakarta.servlet.ServletContext; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.DateUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.web.servlet.ServletContextInitializer; +import org.springframework.stereotype.Component; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; -import javax.servlet.annotation.WebListener; import java.util.Date; import java.util.List; import java.util.Map; @@ -24,20 +26,22 @@ import java.util.concurrent.TimeUnit; /** * @author Administrator */ -@WebListener +@Component @Slf4j @RequiredArgsConstructor -public class StarterListener implements ServletContextListener { +public class StarterListener implements ServletContextInitializer { private final BookService bookService; private final CrawlService crawlService; + private final CrawlParser crawlParser; + @Value("${crawl.update.thread}") private int updateThreadCount; @Override - public void contextInitialized(ServletContextEvent sce) { + public void onStartup(ServletContext servletContext) { for (int i = 0; i < updateThreadCount; i++) { new Thread(() -> { log.info("程序启动,开始执行自动更新线程。。。"); @@ -56,20 +60,24 @@ public class StarterListener implements ServletContextListener { CrawlSource source = crawlService.queryCrawlSource(needUpdateBook.getCrawlSourceId()); RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class); //解析小说基本信息 - CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId(),book -> { + crawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId(), book -> { //这里只做老书更新 book.setId(needUpdateBook.getId()); book.setWordCount(needUpdateBook.getWordCount()); - if (needUpdateBook.getPicUrl() != null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) { + if (needUpdateBook.getPicUrl() != null && needUpdateBook.getPicUrl() + .contains(Constants.LOCAL_PIC_PREFIX)) { //本地图片则不更新 book.setPicUrl(null); } //查询已存在的章节 - Map existBookIndexMap = bookService.queryExistBookIndexMap(needUpdateBook.getId()); + Map existBookIndexMap = bookService.queryExistBookIndexMap( + needUpdateBook.getId()); //解析章节目录 - CrawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(), book, ruleBean, existBookIndexMap,chapter -> { - bookService.updateBookAndIndexAndContent(book, chapter.getBookIndexList(), chapter.getBookContentList(), existBookIndexMap); - }); + crawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(), book, + ruleBean, existBookIndexMap, + chapter -> bookService.updateBookAndIndexAndContent(book, + chapter.getBookIndexList(), + chapter.getBookContentList(), existBookIndexMap), null); }); } catch (Exception e) { log.error(e.getMessage(), e); @@ -88,7 +96,6 @@ public class StarterListener implements ServletContextListener { } - new Thread(() -> { log.info("程序启动,开始执行单本采集任务线程。。。"); while (true) { @@ -102,8 +109,8 @@ public class StarterListener implements ServletContextListener { //查询爬虫规则 CrawlSource source = crawlService.queryCrawlSource(task.getSourceId()); RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class); - - if (crawlService.parseBookAndSave(task.getCatId(), ruleBean, task.getSourceId(), task.getSourceBookId())) { + if (crawlService.parseBookAndSave(task.getCatId(), ruleBean, task.getSourceId(), + task.getSourceBookId(), task)) { //采集成功 crawlStatus = 1; } @@ -116,6 +123,7 @@ public class StarterListener implements ServletContextListener { } catch (Exception e) { log.error(e.getMessage(), e); } + if (task != null) { crawlService.updateCrawlSingleTask(task, crawlStatus); } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/schedule/CrawlThreadMonitor.java b/novel-crawl/src/main/java/com/java2nb/novel/core/schedule/CrawlThreadMonitor.java deleted file mode 100644 index a04ce49..0000000 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/schedule/CrawlThreadMonitor.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.java2nb.novel.core.schedule; - - -import com.java2nb.novel.core.cache.CacheKey; -import com.java2nb.novel.core.cache.CacheService; -import com.java2nb.novel.entity.CrawlSource; -import com.java2nb.novel.service.CrawlService; -import io.github.xxyopen.util.ThreadUtil; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.util.List; -import java.util.Set; - -/** - * 爬虫线程监控器,监控执行完成的爬虫源,并修改状态 - * - * @author Administrator - */ -@Service -@RequiredArgsConstructor -@Slf4j -public class CrawlThreadMonitor { - - private final CacheService cacheService; - - private final CrawlService crawlService; - - @Scheduled(fixedRate = 1000 * 60 * 5) - public void monitor() { - - //查询需要监控的正在运行的爬虫源 - List sources = crawlService.queryCrawlSourceByStatus((byte) 1); - - for (CrawlSource source : sources) { - Set runningCrawlThreadIds = (Set) cacheService.getObject(CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + source.getId()); - boolean sourceStop = true; - if (runningCrawlThreadIds != null) { - for (Long threadId : runningCrawlThreadIds) { - Thread thread = ThreadUtil.findThread(threadId); - - if (thread != null && thread.isAlive()) { - //有活跃线程,说明该爬虫源正在运行,数据库中状态正确,不需要修改 - sourceStop = false; - - } - - } - } - - if (sourceStop) { - crawlService.updateCrawlSourceStatus(source.getId(), (byte) 0); - } - - - } - - } -} diff --git a/novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java b/novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java index fdb36a5..b8a0620 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java @@ -47,13 +47,15 @@ public interface CrawlService { /** * 采集并保存小说 - * @param catId 分类ID - * @param bookId 小说ID - * @param sourceId 源ID + * + * @param catId 分类ID * @param ruleBean 采集规则\ + * @param sourceId 源ID + * @param bookId 小说ID + * @param task * @return true:成功,false:失败 - * */ - boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId); + */ + boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId, CrawlSingleTask task) throws InterruptedException; /** * 根据爬虫状态查询爬虫源集合 @@ -117,4 +119,9 @@ public interface CrawlService { * @return */ CrawlSource getCrawlSource(Integer id); + + /** + * 采集任务进度查询 + * */ + Integer getTaskProgress(Long taskId); } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java index d8b02b8..ad0de60 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java @@ -2,17 +2,10 @@ package com.java2nb.novel.service.impl; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.pagehelper.PageHelper; -import io.github.xxyopen.model.page.PageBean; -import com.java2nb.novel.core.cache.CacheKey; import com.java2nb.novel.core.cache.CacheService; import com.java2nb.novel.core.crawl.CrawlParser; import com.java2nb.novel.core.crawl.RuleBean; import com.java2nb.novel.core.enums.ResponseStatus; -import io.github.xxyopen.model.page.builder.pagehelper.PageBuilder; -import io.github.xxyopen.util.IdWorker; -import io.github.xxyopen.util.ThreadUtil; -import io.github.xxyopen.web.exception.BusinessException; -import io.github.xxyopen.web.util.BeanUtil; import com.java2nb.novel.entity.Book; import com.java2nb.novel.entity.CrawlSingleTask; import com.java2nb.novel.entity.CrawlSource; @@ -22,9 +15,15 @@ import com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport; import com.java2nb.novel.mapper.CrawlSourceMapper; import com.java2nb.novel.service.BookService; import com.java2nb.novel.service.CrawlService; +import com.java2nb.novel.utils.CrawlHttpClient; import com.java2nb.novel.vo.CrawlSingleTaskVO; import com.java2nb.novel.vo.CrawlSourceVO; -import io.github.xxyopen.web.util.SpringUtil; +import io.github.xxyopen.model.page.PageBean; +import io.github.xxyopen.model.page.builder.pagehelper.PageBuilder; +import io.github.xxyopen.util.IdWorker; +import io.github.xxyopen.util.ThreadUtil; +import io.github.xxyopen.web.exception.BusinessException; +import io.github.xxyopen.web.util.BeanUtil; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -33,12 +32,12 @@ import org.mybatis.dynamic.sql.render.RenderingStrategies; import org.mybatis.dynamic.sql.select.render.SelectStatementProvider; import org.springframework.stereotype.Service; +import java.time.Duration; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClientWithChrome; import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*; import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo; import static org.mybatis.dynamic.sql.select.SelectDSL.select; @@ -51,6 +50,7 @@ import static org.mybatis.dynamic.sql.select.SelectDSL.select; @Slf4j public class CrawlServiceImpl implements CrawlService { + private final CrawlParser crawlParser; private final CrawlSourceMapper crawlSourceMapper; @@ -58,10 +58,14 @@ public class CrawlServiceImpl implements CrawlService { private final BookService bookService; - private final CacheService cacheService; - private final IdWorker idWorker = IdWorker.INSTANCE; + private final CrawlHttpClient crawlHttpClient; + + private final Map crawlSourceStatusMap = new HashMap<>(); + + private final Map> runningCrawlThread = new HashMap<>(); + @Override public void addCrawlSource(CrawlSource source) { @@ -71,15 +75,16 @@ public class CrawlServiceImpl implements CrawlService { crawlSourceMapper.insertSelective(source); } + @Override public void updateCrawlSource(CrawlSource source) { - if(source.getId()!=null){ - Optional opt=crawlSourceMapper.selectByPrimaryKey(source.getId()); - if(opt.isPresent()) { - CrawlSource crawlSource =opt.get(); + if (source.getId() != null) { + Optional opt = crawlSourceMapper.selectByPrimaryKey(source.getId()); + if (opt.isPresent()) { + CrawlSource crawlSource = opt.get(); if (crawlSource.getSourceStatus() == (byte) 1) { //关闭 - openOrCloseCrawl(crawlSource.getId(),(byte)0); + openOrCloseCrawl(crawlSource.getId(), (byte) 0); } Date currentDate = new Date(); crawlSource.setUpdateTime(currentDate); @@ -89,15 +94,18 @@ public class CrawlServiceImpl implements CrawlService { } } } + @Override public PageBean listCrawlByPage(int page, int pageSize) { PageHelper.startPage(page, pageSize); SelectStatementProvider render = select(id, sourceName, sourceStatus, createTime, updateTime) - .from(crawlSource) - .orderBy(updateTime) - .build() - .render(RenderingStrategies.MYBATIS3); + .from(crawlSource) + .orderBy(updateTime) + .build() + .render(RenderingStrategies.MYBATIS3); List crawlSources = crawlSourceMapper.selectMany(render); + crawlSources.forEach(crawlSource -> crawlSource.setSourceStatus( + Optional.ofNullable(crawlSourceStatusMap.get(crawlSource.getId())).orElse((byte) 0))); PageBean pageBean = PageBuilder.build(crawlSources); pageBean.setList(BeanUtil.copyList(crawlSources, CrawlSourceVO.class)); return pageBean; @@ -107,13 +115,13 @@ public class CrawlServiceImpl implements CrawlService { @Override public void openOrCloseCrawl(Integer sourceId, Byte sourceStatus) { - //判断是开启还是关闭,如果是关闭,则修改数据库状态后获取该爬虫正在运行的线程集合并全部停止 - //如果是开启,先查询数据库中状态,判断该爬虫源是否还在运行,如果在运行,则忽略, - // 如果没有则修改数据库状态,并启动线程爬取小说数据加入到runningCrawlThread中 + // 判断是开启还是关闭,如果是关闭,则获取该爬虫源正在运行的线程集合并全部中断 + // 如果是开启,先判断该爬虫源是否还在运行,如果在运行,则忽略,如果没有运行则启动线程爬取小说数据并加入到runningCrawlThread中 + // 最后,保存爬虫源状态 if (sourceStatus == (byte) 0) { - //关闭,直接修改数据库状态,并直接修改数据库状态后获取该爬虫正在运行的线程集合全部停止 - SpringUtil.getBean(CrawlService.class).updateCrawlSourceStatus(sourceId, sourceStatus); - Set runningCrawlThreadId = (Set) cacheService.getObject(CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + sourceId); + // 关闭 + // 将该爬虫源正在运行的线程集合全部停止 + Set runningCrawlThreadId = runningCrawlThread.get(sourceId); if (runningCrawlThreadId != null) { for (Long ThreadId : runningCrawlThreadId) { Thread thread = ThreadUtil.findThread(ThreadId); @@ -125,16 +133,13 @@ public class CrawlServiceImpl implements CrawlService { } else { - //开启 - //查询爬虫源状态和规则 - CrawlSource source = queryCrawlSource(sourceId); - Byte realSourceStatus = source.getSourceStatus(); - + // 开启 + Byte realSourceStatus = Optional.ofNullable(crawlSourceStatusMap.get(sourceId)).orElse((byte) 0); if (realSourceStatus == (byte) 0) { - //该爬虫源已经停止运行了,修改数据库状态,并启动线程爬取小说数据加入到runningCrawlThread中 - SpringUtil.getBean(CrawlService.class).updateCrawlSourceStatus(sourceId, sourceStatus); + // 查询爬虫源规则 + CrawlSource source = queryCrawlSource(sourceId); + //该爬虫源已经停止运行了,启动线程爬取小说数据并将线程加入到runningCrawlThread中 RuleBean ruleBean = new ObjectMapper().readValue(source.getCrawlRule(), RuleBean.class); - Set threadIds = new HashSet<>(); //按分类开始爬虫解析任务 for (int i = 1; i < 8; i++) { @@ -143,25 +148,25 @@ public class CrawlServiceImpl implements CrawlService { thread.start(); //thread加入到监控缓存中 threadIds.add(thread.getId()); - } - cacheService.setObject(CacheKey.RUNNING_CRAWL_THREAD_KEY_PREFIX + sourceId, threadIds); - - + runningCrawlThread.put(sourceId, threadIds); } - } + // 保存爬虫源状态 + crawlSourceStatusMap.put(sourceId, sourceStatus); + } @Override public CrawlSource queryCrawlSource(Integer sourceId) { - SelectStatementProvider render = select(CrawlSourceDynamicSqlSupport.sourceStatus, CrawlSourceDynamicSqlSupport.crawlRule) - .from(crawlSource) - .where(id, isEqualTo(sourceId)) - .build() - .render(RenderingStrategies.MYBATIS3); + SelectStatementProvider render = select(CrawlSourceDynamicSqlSupport.sourceStatus, + CrawlSourceDynamicSqlSupport.crawlRule) + .from(crawlSource) + .where(id, isEqualTo(sourceId)) + .build() + .render(RenderingStrategies.MYBATIS3); return crawlSourceMapper.selectMany(render).get(0); } @@ -182,13 +187,23 @@ public class CrawlServiceImpl implements CrawlService { public PageBean listCrawlSingleTaskByPage(int page, int pageSize) { PageHelper.startPage(page, pageSize); SelectStatementProvider render = select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns()) - .from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask) - .orderBy(CrawlSingleTaskDynamicSqlSupport.createTime.descending()) - .build() - .render(RenderingStrategies.MYBATIS3); + .from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask) + .orderBy(CrawlSingleTaskDynamicSqlSupport.createTime.descending()) + .build() + .render(RenderingStrategies.MYBATIS3); List crawlSingleTasks = crawlSingleTaskMapper.selectMany(render); PageBean pageBean = PageBuilder.build(crawlSingleTasks); pageBean.setList(BeanUtil.copyList(crawlSingleTasks, CrawlSingleTaskVO.class)); + for (CrawlSingleTask crawlSingleTask : pageBean.getList()) { + if (crawlSingleTask.getTaskStatus() == 2 + && crawlParser.getCrawlTaskProgress(crawlSingleTask.getId()) != null) { + // 如果排队中的任务有任务进度,将排队中的任务状态修改成采集中并设置任务进度 + crawlSingleTask.setTaskStatus((byte) 3); + crawlSingleTask.setCrawlChapters(crawlParser.getCrawlTaskProgress(crawlSingleTask.getId())); + // 只会有一个任务在采集中 + break; + } + } return pageBean; } @@ -200,7 +215,8 @@ public class CrawlServiceImpl implements CrawlService { @Override public CrawlSingleTask getCrawlSingleTask() { - List list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns()) + List list = crawlSingleTaskMapper.selectMany( + select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns()) .from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask) .where(CrawlSingleTaskDynamicSqlSupport.taskStatus, isEqualTo((byte) 2)) .orderBy(CrawlSingleTaskDynamicSqlSupport.createTime) @@ -217,21 +233,27 @@ public class CrawlServiceImpl implements CrawlService { excCount += 1; task.setExcCount(excCount); if (status == 1 || excCount == 5) { - //当采集成功或者采集次数等于5,则更新采集最终状态,并停止采集 + // 当采集成功或者采集次数等于5,则更新采集最终状态,并停止采集 task.setTaskStatus(status); } + if (status == 1) { + // 当采集成功,保存采集的章节数量 + task.setCrawlChapters(crawlParser.getCrawlTaskProgress(task.getId())); + } crawlSingleTaskMapper.updateByPrimaryKeySelective(task); + // 删除任务进度 + crawlParser.removeCrawlTaskProgress(task.getId()); } @Override public CrawlSource getCrawlSource(Integer id) { - Optional opt=crawlSourceMapper.selectByPrimaryKey(id); - if(opt.isPresent()) { - CrawlSource crawlSource =opt.get(); - return crawlSource; - } - return null; + return crawlSourceMapper.selectByPrimaryKey(id).orElse(null); + } + + @Override + public Integer getTaskProgress(Long taskId) { + return Optional.ofNullable(crawlParser.getCrawlTaskProgress(taskId)).orElse(0); } /** @@ -240,6 +262,11 @@ public class CrawlServiceImpl implements CrawlService { @Override public void parseBookList(int catId, RuleBean ruleBean, Integer sourceId) { + String catIdRule = ruleBean.getCatIdRule().get("catId" + catId); + if (StringUtils.isBlank(catIdRule)) { + return; + } + //当前页码1 int page = 1; int totalPage = page; @@ -247,66 +274,95 @@ public class CrawlServiceImpl implements CrawlService { while (page <= totalPage) { try { + String catBookListUrl; + if (StringUtils.isNotBlank(ruleBean.getBookListUrl())) { + // 兼容老规则 + // 拼接分类URL + catBookListUrl = ruleBean.getBookListUrl() + .replace("{catId}", catIdRule) + .replace("{page}", page + ""); + } else { + // 新规则 + // 拼接分类URL + catBookListUrl = catIdRule.replace("{page}", page + ""); + } + log.info("catBookListUrl:{}", catBookListUrl); - if (StringUtils.isNotBlank(ruleBean.getCatIdRule().get("catId" + catId))) { - //拼接分类URL - String catBookListUrl = ruleBean.getBookListUrl() - .replace("{catId}", ruleBean.getCatIdRule().get("catId" + catId)) - .replace("{page}", page + ""); - - String bookListHtml = getByHttpClientWithChrome(catBookListUrl); - if (bookListHtml != null) { - Pattern bookIdPatten = Pattern.compile(ruleBean.getBookIdPatten()); - Matcher bookIdMatcher = bookIdPatten.matcher(bookListHtml); - boolean isFindBookId = bookIdMatcher.find(); - while (isFindBookId) { - try { - //1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时) - //捕获中断异常InterruptedException来退出线程。 - //2.非阻塞过程中通过判断中断标志来退出线程。 - if (Thread.currentThread().isInterrupted()) { - return; - } - - - String bookId = bookIdMatcher.group(1); - parseBookAndSave(catId, ruleBean, sourceId, bookId); - } catch (Exception e) { - log.error(e.getMessage(), e); + String bookListHtml = crawlHttpClient.get(catBookListUrl, ruleBean.getCharset()); + if (bookListHtml != null) { + Pattern bookIdPatten = Pattern.compile(ruleBean.getBookIdPatten()); + Matcher bookIdMatcher = bookIdPatten.matcher(bookListHtml); + boolean isFindBookId = bookIdMatcher.find(); + while (isFindBookId) { + try { + //1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时) + //捕获中断异常InterruptedException来退出线程。 + //2.非阻塞过程中通过判断中断标志来退出线程。 + if (Thread.currentThread().isInterrupted()) { + return; } - - isFindBookId = bookIdMatcher.find(); + String bookId = bookIdMatcher.group(1); + parseBookAndSave(catId, ruleBean, sourceId, bookId, null); + } catch (InterruptedException e) { + log.error(e.getMessage(), e); + //1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时) + //捕获中断异常InterruptedException来退出线程。 + //2.非阻塞过程中通过判断中断标志来退出线程。 + return; + } catch (Exception e) { + log.error(e.getMessage(), e); } - Pattern totalPagePatten = Pattern.compile(ruleBean.getTotalPagePatten()); - Matcher totalPageMatcher = totalPagePatten.matcher(bookListHtml); - boolean isFindTotalPage = totalPageMatcher.find(); - if (isFindTotalPage) { + isFindBookId = bookIdMatcher.find(); + } - totalPage = Integer.parseInt(totalPageMatcher.group(1)); - - } + Pattern totalPagePatten = Pattern.compile(ruleBean.getTotalPagePatten()); + Matcher totalPageMatcher = totalPagePatten.matcher(bookListHtml); + boolean isFindTotalPage = totalPageMatcher.find(); + if (isFindTotalPage) { + totalPage = Integer.parseInt(totalPageMatcher.group(1)); } } + } catch (InterruptedException e) { + log.error(e.getMessage(), e); + //1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时) + //捕获中断异常InterruptedException来退出线程。 + //2.非阻塞过程中通过判断中断标志来退出线程。 + return; } catch (Exception e) { log.error(e.getMessage(), e); } - - page += 1; + if (page >= totalPage) { + // 第一遍采集完成,翻到第一页,继续第二次采集,适用于分页数比较少的最近更新列表 + page = 1; + try { + // 第一遍采集完成,休眠1分钟 + Thread.sleep(Duration.ofMinutes(1)); + } catch (InterruptedException e) { + log.error(e.getMessage(), e); + //1.阻塞过程(使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时) + //捕获中断异常InterruptedException来退出线程。 + //2.非阻塞过程中通过判断中断标志来退出线程。 + return; + } + } else { + page += 1; + } } } @Override - public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId) { + public boolean parseBookAndSave(int catId, RuleBean ruleBean, Integer sourceId, String bookId, CrawlSingleTask task) + throws InterruptedException { final AtomicBoolean parseResult = new AtomicBoolean(false); - CrawlParser.parseBook(ruleBean, bookId, book -> { + crawlParser.parseBook(ruleBean, bookId, book -> { if (book.getBookName() == null || book.getAuthorName() == null) { return; } @@ -330,9 +386,11 @@ public class CrawlServiceImpl implements CrawlService { book.setCrawlLastTime(new Date()); book.setId(idWorker.nextId()); //解析章节目录 - boolean parseIndexContentResult = CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0), chapter -> { - bookService.saveBookAndIndexAndContent(book, chapter.getBookIndexList(), chapter.getBookContentList()); - }); + boolean parseIndexContentResult = crawlParser.parseBookIndexAndContent(bookId, book, ruleBean, + new HashMap<>(0), chapter -> { + bookService.saveBookAndIndexAndContent(book, chapter.getBookIndexList(), + chapter.getBookContentList()); + }, task); parseResult.set(parseIndexContentResult); } else { @@ -356,11 +414,13 @@ public class CrawlServiceImpl implements CrawlService { @Override public List queryCrawlSourceByStatus(Byte sourceStatus) { - SelectStatementProvider render = select(CrawlSourceDynamicSqlSupport.id, CrawlSourceDynamicSqlSupport.sourceStatus, CrawlSourceDynamicSqlSupport.crawlRule) - .from(crawlSource) - .where(CrawlSourceDynamicSqlSupport.sourceStatus, isEqualTo(sourceStatus)) - .build() - .render(RenderingStrategies.MYBATIS3); + SelectStatementProvider render = select(CrawlSourceDynamicSqlSupport.id, + CrawlSourceDynamicSqlSupport.sourceStatus, CrawlSourceDynamicSqlSupport.crawlRule) + .from(crawlSource) + .where(CrawlSourceDynamicSqlSupport.sourceStatus, isEqualTo(sourceStatus)) + .build() + .render(RenderingStrategies.MYBATIS3); return crawlSourceMapper.selectMany(render); } + } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/utils/Constants.java b/novel-crawl/src/main/java/com/java2nb/novel/utils/Constants.java index d278c19..b84f91a 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/utils/Constants.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/utils/Constants.java @@ -7,7 +7,7 @@ public class Constants { /** * 本地图片保存前缀 - * */ + */ public static final String LOCAL_PIC_PREFIX = "/localPic/"; /** @@ -23,5 +23,10 @@ public class Constants { /** * 爬取小说http请求失败重试次数 */ - public static final Integer HTTP_FAIL_RETRY_COUNT = 5; + public static final Integer HTTP_FAIL_RETRY_COUNT = 3; + + /** + * 爬虫默认编码 + */ + public static final String CRAWL_DEFAULT_CHARSET = "UTF-8"; } diff --git a/novel-crawl/src/main/java/com/java2nb/novel/utils/CrawlHttpClient.java b/novel-crawl/src/main/java/com/java2nb/novel/utils/CrawlHttpClient.java new file mode 100644 index 0000000..cdc90f0 --- /dev/null +++ b/novel-crawl/src/main/java/com/java2nb/novel/utils/CrawlHttpClient.java @@ -0,0 +1,53 @@ +package com.java2nb.novel.utils; + +import com.java2nb.novel.core.utils.HttpUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Objects; +import java.util.Random; + +/** + * @author Administrator + */ +@Slf4j +@Component +public class CrawlHttpClient { + + @Value("${crawl.interval.min}") + private Integer intervalMin; + + @Value("${crawl.interval.max}") + private Integer intervalMax; + + private final Random random = new Random(); + + private static final ThreadLocal RETRY_COUNT = new ThreadLocal<>(); + + public String get(String url, String charset) throws InterruptedException { + if (Objects.nonNull(intervalMin) && Objects.nonNull(intervalMax) && intervalMax > intervalMin) { + Thread.sleep(random.nextInt(intervalMax - intervalMin + 1) + intervalMin); + } + String body = HttpUtil.getByHttpClientWithChrome(url, charset); + if (Objects.isNull(body) || body.length() < Constants.INVALID_HTML_LENGTH) { + return processErrorHttpResult(url, charset); + } + //成功获得html内容 + return body; + } + + private String processErrorHttpResult(String url, String charset) throws InterruptedException{ + Integer count = RETRY_COUNT.get(); + if (count == null) { + count = 0; + } + if (count < Constants.HTTP_FAIL_RETRY_COUNT) { + RETRY_COUNT.set(++count); + return get(url, charset); + } + RETRY_COUNT.remove(); + return null; + } + +} diff --git a/novel-crawl/src/main/resources/application-dev.yml b/novel-crawl/src/main/resources/application-dev.yml index 4ea9fe8..f53bb81 100644 --- a/novel-crawl/src/main/resources/application-dev.yml +++ b/novel-crawl/src/main/resources/application-dev.yml @@ -1,3 +1,3 @@ spring: - profiles: - include: [common-dev] \ No newline at end of file + config: + import: classpath:application-common-dev.yml \ No newline at end of file diff --git a/novel-crawl/src/main/resources/application-prod.yml b/novel-crawl/src/main/resources/application-prod.yml index 54ce802..14d0afa 100644 --- a/novel-crawl/src/main/resources/application-prod.yml +++ b/novel-crawl/src/main/resources/application-prod.yml @@ -1,3 +1,3 @@ spring: - profiles: - include: [common-prod] \ No newline at end of file + config: + import: classpath:application-common-prod.yml \ No newline at end of file diff --git a/novel-crawl/src/main/resources/application.yml b/novel-crawl/src/main/resources/application.yml index 132a14e..b4908fe 100644 --- a/novel-crawl/src/main/resources/application.yml +++ b/novel-crawl/src/main/resources/application.yml @@ -14,12 +14,18 @@ admin: username: admin password: admin -#爬虫自动更新的线程数 -#建议小说数量不多或者正在运行新书入库爬虫的情况下设置为1即可 -#随着小说数量的增多可以逐渐增加,但建议不要超出CPU的线程数 + + crawl: update: + #爬虫自动更新的线程数 + #建议小说数量不多或者正在运行新书入库爬虫的情况下设置为1即可 + #随着小说数量的增多可以逐渐增加,但建议不要超出CPU的线程数 thread: 1 + # 采集间隔时间,单位:毫秒 + interval: + min: 300 + max: 500 diff --git a/novel-crawl/src/main/resources/logback-boot.xml b/novel-crawl/src/main/resources/logback-boot.xml index 05c7c83..2691bfd 100644 --- a/novel-crawl/src/main/resources/logback-boot.xml +++ b/novel-crawl/src/main/resources/logback-boot.xml @@ -12,9 +12,6 @@ - ${CONSOLE_LOG_PATTERN} UTF-8 @@ -22,43 +19,43 @@ - - + + - - logs/novel-crawl.log + + logs/novel-crawl.log - - + + - - logs/debug.%d.%i.log + + logs/debug.%d{yyyy-MM-dd}.%i.log 30 - - - 10MB - + + 10MB + + 1GB - - %d %p (%file:%line\)- %m%n - + %d %p (%file:%line\)- %m%n UTF-8 + + - + - + \ No newline at end of file diff --git a/novel-crawl/src/main/resources/favicon.ico b/novel-crawl/src/main/resources/static/favicon.ico similarity index 100% rename from novel-crawl/src/main/resources/favicon.ico rename to novel-crawl/src/main/resources/static/favicon.ico diff --git a/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_add.html b/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_add.html index ec33b81..693c91c 100644 --- a/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_add.html +++ b/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_add.html @@ -110,7 +110,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html b/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html index d7fe935..f0bca00 100644 --- a/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html +++ b/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html @@ -48,6 +48,9 @@ 采集小说作者名 + + 采集进度 + 采集次数 @@ -113,9 +116,15 @@ diff --git a/novel-front/src/main/resources/templates/author/index.html b/novel-front/src/main/resources/templates/author/index.html index b843bab..00a6c63 100644 --- a/novel-front/src/main/resources/templates/author/index.html +++ b/novel-front/src/main/resources/templates/author/index.html @@ -142,10 +142,12 @@ diff --git a/novel-front/src/main/resources/templates/book/book_comment_reply.html b/novel-front/src/main/resources/templates/book/book_comment_reply.html new file mode 100644 index 0000000..0774fe5 --- /dev/null +++ b/novel-front/src/main/resources/templates/book/book_comment_reply.html @@ -0,0 +1,216 @@ + + + + + + + + + + + +
+
+ +
+
+ +
+
+
+ +
+
+
+

评论回复区

(0条) +
+ 发表回复 +
+
+
+ +
+ +
+
+
+ +
+
+ 发表回复 + + +
+ +
+ 0/1000 + 发表 +
+
+
+ +
+
+
+ + + + + +
+
+ +
+
+
+ + + + diff --git a/novel-front/src/main/resources/templates/book/book_detail.html b/novel-front/src/main/resources/templates/book/book_detail.html index ea1d75f..8028f32 100644 --- a/novel-front/src/main/resources/templates/book/book_detail.html +++ b/novel-front/src/main/resources/templates/book/book_detail.html @@ -22,21 +22,23 @@
+ th:attr="alt=${book.bookName}"/>

+ th:utext="${book.authorName}+' 著'">
  • 类别: - 状态:连载中已完结 + 状态:连载中已完结 总点击: 总字数:
@@ -70,7 +72,9 @@
  • - +
  • @@ -86,21 +90,46 @@
    -

    作品评论区

    +

    作品评论区

    发表评论
    -
    +
    暂无评论
    -
    -
    + -
    + @@ -109,7 +138,7 @@ 发表评论
    @@ -175,10 +204,12 @@
  • - +
    - + @@ -210,7 +241,7 @@ var bookId = pathname.substring(pathname.lastIndexOf("/") + 1, pathname.lastIndexOf(".")) //查询章节信息 var lastBookIndexId = $("#lastBookIndexId").val(); - if(lastBookIndexId){ + if (lastBookIndexId) { $.ajax({ type: "get", url: "/book/queryBookIndexAbout", @@ -232,7 +263,7 @@ layer.alert('网络异常'); } }) - }else{ + } else { $("#optBtn").remove(); } @@ -264,9 +295,6 @@ }) - - - var currentBId = 37, spmymoney = 0; var relationStep = 0; var authorUId = 8; @@ -283,7 +311,6 @@ }); - $("#AuthorOtherNovel li").unbind("mouseover"); $('#txtComment').on('input propertychange', function () { @@ -301,7 +328,7 @@ }); - function loadCommentList(){ + function loadCommentList() { $.ajax({ type: "get", url: "/book/listCommentByPage", @@ -311,20 +338,25 @@ if (data.code == 200) { var commentList = data.data.list; if (commentList.length > 0) { - $("#bookCommentTotal").html("("+data.data.total+"条)"); + $("#bookCommentTotal").html("(" + data.data.total + "条)"); var commentListHtml = ""; for (var i = 0; i < commentList.length; i++) { var comment = commentList[i]; commentListHtml += ("
    " + "
    " + - "\"\"" + + "\"\"" + "见习
    " + - "
      \t\t\t
    • "+(comment.createUserName)+"
    • " + - comment.commentContent+ + "
        \t\t\t
      • " + (comment.createUserName) + "" + (comment.location ? comment.location + "读者" : '') + "
      • " + + comment.commentContent + "
      • " + - ""+comment.createTime+"" + - "(0)" + - "
      • \t\t
      \t
    "); + "" + comment.createTime + "" + + "("+comment.unLikesCount+")" + + "("+comment.likesCount+")" + + "回复("+comment.replyCount+ + ")" + + "
  • \t\t
\t
" + ) + ; } $("#commentPanel").html(commentListHtml); $("#noCommentPanel").hide(); @@ -348,6 +380,55 @@ } }) } + function toggleCommentLike(commentId) { + $.ajax({ + type: "post", + url: "/book/toggleCommentLike", + data: {'commentId': commentId}, + dataType: "json", + success: function (data) { + if (data.code == 200) { + $("#likeCount"+commentId).text("("+data.data+")") + } else if (data.code == 1001) { + //未登录 + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); + + } else { + layer.alert(data.msg); + } + + }, + error: function () { + layer.alert('网络异常'); + } + }) + + } + + function toggleCommentUnLike(commentId) { + $.ajax({ + type: "post", + url: "/book/toggleCommentUnLike", + data: {'commentId': commentId}, + dataType: "json", + success: function (data) { + if (data.code == 200) { + $("#unLikeCount"+commentId).text("("+data.data+")") + } else if (data.code == 1001) { + //未登录 + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); + + } else { + layer.alert(data.msg); + } + + }, + error: function () { + layer.alert('网络异常'); + } + }) + + } diff --git a/novel-front/src/main/resources/templates/book/bookclass.html b/novel-front/src/main/resources/templates/book/bookclass.html index a44cea9..afa7291 100644 --- a/novel-front/src/main/resources/templates/book/bookclass.html +++ b/novel-front/src/main/resources/templates/book/bookclass.html @@ -6,8 +6,8 @@ - - + + diff --git a/novel-front/src/main/resources/templates/index.html b/novel-front/src/main/resources/templates/index.html index 53f6627..1e48ad7 100644 --- a/novel-front/src/main/resources/templates/index.html +++ b/novel-front/src/main/resources/templates/index.html @@ -7,8 +7,8 @@ - - + + diff --git a/novel-front/src/main/resources/templates/mobile/pay/index.html b/novel-front/src/main/resources/templates/mobile/pay/index.html index befa955..98a0a97 100644 --- a/novel-front/src/main/resources/templates/mobile/pay/index.html +++ b/novel-front/src/main/resources/templates/mobile/pay/index.html @@ -114,7 +114,7 @@ style="color: #3eaf7c" id="accountBalance">10 屋币
-
+
选择充值金额
  • diff --git a/novel-front/src/main/resources/templates/mobile/user/favorites.html b/novel-front/src/main/resources/templates/mobile/user/favorites.html index 44d3d44..3630485 100644 --- a/novel-front/src/main/resources/templates/mobile/user/favorites.html +++ b/novel-front/src/main/resources/templates/mobile/user/favorites.html @@ -20,6 +20,22 @@ - @@ -95,6 +121,10 @@
    +
    @@ -105,11 +135,12 @@ -
    diff --git a/novel-front/src/main/resources/templates/user/favorites.html b/novel-front/src/main/resources/templates/user/favorites.html index 985bec0..08ebb60 100644 --- a/novel-front/src/main/resources/templates/user/favorites.html +++ b/novel-front/src/main/resources/templates/user/favorites.html @@ -53,7 +53,7 @@ 更新时间 - 书签 + 操作 @@ -94,7 +94,7 @@ var bookShelfListHtml = ""; for (var i = 0; i < bookShelfList.length; i++) { var book = bookShelfList[i]; - bookShelfListHtml += (" \n" + + bookShelfListHtml += (" \n" + " \n" + " [" + book.catName + "]\n" + " \n" + @@ -109,7 +109,8 @@ " " + book.lastIndexUpdateTime + "\n" + " \n" + " \n" + - "继续阅读" + + "" + + "" + " \n" + " "); } @@ -160,5 +161,21 @@ }) } + + function removeFromBookShelf(bookId) { + + $.ajax({ + type: "delete", + url: "/user/removeFromBookShelf/" + bookId, + data: {}, + dataType: "json", + success: function (data) { + if (data.code == 200) { + $("#shelf" + bookId).remove(); + } + } + }); + + } diff --git a/novel-front/src/main/resources/templates/user/login.html b/novel-front/src/main/resources/templates/user/login.html index 2420d9c..e22aafc 100644 --- a/novel-front/src/main/resources/templates/user/login.html +++ b/novel-front/src/main/resources/templates/user/login.html @@ -17,7 +17,7 @@
    -

    +

    • diff --git a/pom.xml b/pom.xml index 3aacc45..d5e565e 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.java2nb novel - 4.3.0 + 5.2.0 novel-common novel-front @@ -20,32 +20,30 @@ org.springframework.boot spring-boot-starter-parent - 2.1.18.RELEASE + 3.4.0 UTF-8 UTF-8 - 1.8 + 21 true 8.0.29 - 2.1.4 + 3.0.4 1.4.0 - 1.1.4 + 1.5.0 1.4.6 - 1.0.2 3.4 0.9.1 6.3.1 1.4.1.RELEASE 3.12.5 - 3.0.0 + 5.5.1 3.16.3 1.5 4.35.139.ALL 1.0.0 - 2.15.1 @@ -55,6 +53,13 @@ novel-common ${project.version} + + org.springframework.ai + spring-ai-bom + 1.0.0-M6 + pom + import + @@ -108,4 +113,35 @@ + + + + + central-repo + + + central + https://repo.maven.apache.org/maven2/ + + true + + + false + + + + + + central-plugin + https://repo.maven.apache.org/maven2/ + + true + + + false + + + + + diff --git a/templates/dark/html/book/book_detail.html b/templates/dark/html/book/book_detail.html index cddc15f..b48a5cd 100644 --- a/templates/dark/html/book/book_detail.html +++ b/templates/dark/html/book/book_detail.html @@ -260,7 +260,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/dark/html/mobile/book/book_detail.html b/templates/dark/html/mobile/book/book_detail.html index fc20f58..41790fb 100644 --- a/templates/dark/html/mobile/book/book_detail.html +++ b/templates/dark/html/mobile/book/book_detail.html @@ -260,7 +260,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/green/html/author/content_add.html b/templates/green/html/author/content_add.html index 183bee0..2754cad 100644 --- a/templates/green/html/author/content_add.html +++ b/templates/green/html/author/content_add.html @@ -8,6 +8,79 @@ 作家管理系统-小说精品屋 + @@ -46,18 +119,29 @@
      • 章节名: -
      • - 章节内容:
      • -

      • +
      • + 章节内容: +
      • + - 是否收费: -
      • 免费 - 收费
      • + 是否收费: +
      • 免费 + 收费 +
      • -
      • +
      • @@ -113,9 +197,10 @@ var lock = false; + function addBookContent() { - if(lock){ + if (lock) { return; } lock = true; @@ -125,14 +210,14 @@ var indexName = $("#bookIndex").val(); - if(!indexName){ + if (!indexName) { $("#LabErr").html("章节名不能为空!"); lock = false; return; } var content = $("#bookContent").val(); - if(!content){ + if (!content) { $("#LabErr").html("章节内容不能为空!"); lock = false; return; @@ -142,17 +227,15 @@ var isVip = $("input:checked[name=isVip]").val(); - - $.ajax({ type: "POST", url: "/author/addBookContent", - data: {'bookId':bookId,'indexName':indexName,'content':content,'isVip':isVip}, + data: {'bookId': bookId, 'indexName': indexName, 'content': content, 'isVip': isVip}, dataType: "json", success: function (data) { if (data.code == 200) { - window.location.href = '/author/index_list.html?bookId='+bookId; + window.location.href = '/author/index_list.html?bookId=' + bookId; } else { @@ -169,5 +252,116 @@ } + + // 打字机效果函数 + function typeWriter(textarea, text, speed = 50) { + let i = 0; + const timer = setInterval(() => { + if (i < text.length) { + textarea.val(textarea.val() + text.charAt(i)); + i++; + // 滚动到底部 + textarea.scrollTop(textarea[0].scrollHeight); + } else { + clearInterval(timer); + } + }, speed); + } + + $('.ai-toolbar .ai-link').click(function (e) { + e.preventDefault(); // 阻止默认链接行为 + const type = $(this).data('type'); + const textarea = $('#bookContent'); + const selectedText = textarea.val().substring(textarea[0].selectionStart, textarea[0].selectionEnd); + + // 检查是否选中文本 + if (!selectedText) { + layer.msg('请先选中要处理的文本'); + return; + } + + const loading = layer.load(1, {shade: 0.3}); + + // 参数配置 + let params = {text: selectedText}; + if (type === 'stream/expand' || type === 'stream/condense') { + layer.prompt({ + title: '请输入比例', + value: 2, + btn: ['确定', '取消'], + btn2: function () { + layer.close(loading); + }, + cancel: function () { + layer.close(loading); + } + }, function (value, index) { + if (isNaN(Number(value)) || isNaN(parseFloat(value))) { + layer.msg('请输入正确的比例'); + return; + } + if (type === 'stream/expand' && value <= 1) { + layer.msg('请输入正确的比例'); + return; + } + if (type === 'stream/condense' && (value <= 0 || value >= 1)) { + layer.msg('请输入正确的比例'); + return; + } + params.ratio = parseFloat(value) * 100; + layer.close(index); + sendRequest(type, params, loading, textarea); + }); + return; + } else if (type === 'stream/continue') { + layer.prompt({ + title: '请输入续写长度(字数)', + value: 200, + btn: ['确定', '取消'], + btn2: function () { + layer.close(loading); + }, + cancel: function () { + layer.close(loading); + } + }, function (value, index) { + if (!Number.isInteger(Number(value)) || value <= 0) { + layer.msg('请输入正确的长度'); + return; + } + params.length = parseInt(value); + layer.close(index); + sendRequest(type, params, loading, textarea); + }); + return; + } + + sendRequest(type, params, loading, textarea); + }); + + function sendRequest(type, params, loading, textarea) { + const url = `/author/ai/${type}?text=${encodeURIComponent(params.text)}&ratio=${params.ratio}&length=${params.length}`; + const eventSource = new EventSource(url); + + // 监听消息事件 + eventSource.onmessage = function (event) { + layer.close(loading); + const data = event.data; + console.log('Received data:', data); + + textarea.val(textarea.val() + data); + // 滚动到底部 + textarea.scrollTop(textarea[0].scrollHeight); + }; + + // 监听错误事件 + eventSource.onerror = function (error) { + layer.close(loading); + console.error('EventSource failed:', error); + eventSource.close(); // 关闭连接 + }; + + } + diff --git a/templates/green/html/author/index.html b/templates/green/html/author/index.html index b843bab..00a6c63 100644 --- a/templates/green/html/author/index.html +++ b/templates/green/html/author/index.html @@ -142,10 +142,12 @@ diff --git a/templates/green/html/book/book_detail.html b/templates/green/html/book/book_detail.html index ea1d75f..8028f32 100644 --- a/templates/green/html/book/book_detail.html +++ b/templates/green/html/book/book_detail.html @@ -22,21 +22,23 @@
        + th:attr="alt=${book.bookName}"/>

        + th:utext="${book.authorName}+' 著'">
        • 类别: - 状态:连载中已完结 + 状态:连载中已完结 总点击: 总字数:
        @@ -70,7 +72,9 @@
        • - +
        • @@ -86,21 +90,46 @@
          -

          作品评论区

          +

          作品评论区

          发表评论
          -
          +
          暂无评论
          -
          -
          + -
          + @@ -109,7 +138,7 @@ 发表评论
          @@ -175,10 +204,12 @@
        • - +
          - + @@ -210,7 +241,7 @@ var bookId = pathname.substring(pathname.lastIndexOf("/") + 1, pathname.lastIndexOf(".")) //查询章节信息 var lastBookIndexId = $("#lastBookIndexId").val(); - if(lastBookIndexId){ + if (lastBookIndexId) { $.ajax({ type: "get", url: "/book/queryBookIndexAbout", @@ -232,7 +263,7 @@ layer.alert('网络异常'); } }) - }else{ + } else { $("#optBtn").remove(); } @@ -264,9 +295,6 @@ }) - - - var currentBId = 37, spmymoney = 0; var relationStep = 0; var authorUId = 8; @@ -283,7 +311,6 @@ }); - $("#AuthorOtherNovel li").unbind("mouseover"); $('#txtComment').on('input propertychange', function () { @@ -301,7 +328,7 @@ }); - function loadCommentList(){ + function loadCommentList() { $.ajax({ type: "get", url: "/book/listCommentByPage", @@ -311,20 +338,25 @@ if (data.code == 200) { var commentList = data.data.list; if (commentList.length > 0) { - $("#bookCommentTotal").html("("+data.data.total+"条)"); + $("#bookCommentTotal").html("(" + data.data.total + "条)"); var commentListHtml = ""; for (var i = 0; i < commentList.length; i++) { var comment = commentList[i]; commentListHtml += ("
          " + "
          " + - "\"\"" + + "\"\"" + "见习
          " + - "
            \t\t\t
          • "+(comment.createUserName)+"
          • " + - comment.commentContent+ + "
              \t\t\t
            • " + (comment.createUserName) + "" + (comment.location ? comment.location + "读者" : '') + "
            • " + + comment.commentContent + "
            • " + - ""+comment.createTime+"" + - "(0)" + - "
            • \t\t
            \t
          "); + "" + comment.createTime + "" + + "("+comment.unLikesCount+")" + + "("+comment.likesCount+")" + + "回复("+comment.replyCount+ + ")" + + "
        • \t\t
        \t
        " + ) + ; } $("#commentPanel").html(commentListHtml); $("#noCommentPanel").hide(); @@ -348,6 +380,55 @@ } }) } + function toggleCommentLike(commentId) { + $.ajax({ + type: "post", + url: "/book/toggleCommentLike", + data: {'commentId': commentId}, + dataType: "json", + success: function (data) { + if (data.code == 200) { + $("#likeCount"+commentId).text("("+data.data+")") + } else if (data.code == 1001) { + //未登录 + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); + + } else { + layer.alert(data.msg); + } + + }, + error: function () { + layer.alert('网络异常'); + } + }) + + } + + function toggleCommentUnLike(commentId) { + $.ajax({ + type: "post", + url: "/book/toggleCommentUnLike", + data: {'commentId': commentId}, + dataType: "json", + success: function (data) { + if (data.code == 200) { + $("#unLikeCount"+commentId).text("("+data.data+")") + } else if (data.code == 1001) { + //未登录 + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); + + } else { + layer.alert(data.msg); + } + + }, + error: function () { + layer.alert('网络异常'); + } + }) + + } diff --git a/templates/green/html/book/bookclass.html b/templates/green/html/book/bookclass.html index a44cea9..afa7291 100644 --- a/templates/green/html/book/bookclass.html +++ b/templates/green/html/book/bookclass.html @@ -6,8 +6,8 @@ - - + + diff --git a/templates/green/html/index.html b/templates/green/html/index.html index 53f6627..1e48ad7 100644 --- a/templates/green/html/index.html +++ b/templates/green/html/index.html @@ -7,8 +7,8 @@ - - + + diff --git a/templates/green/html/mobile/pay/index.html b/templates/green/html/mobile/pay/index.html index befa955..98a0a97 100644 --- a/templates/green/html/mobile/pay/index.html +++ b/templates/green/html/mobile/pay/index.html @@ -114,7 +114,7 @@ style="color: #3eaf7c" id="accountBalance">10 屋币
        -
        +
        选择充值金额
      • diff --git a/templates/green/html/mobile/user/favorites.html b/templates/green/html/mobile/user/favorites.html index 44d3d44..3630485 100644 --- a/templates/green/html/mobile/user/favorites.html +++ b/templates/green/html/mobile/user/favorites.html @@ -20,6 +20,22 @@ - @@ -95,6 +121,10 @@
        +
        @@ -105,11 +135,12 @@ -
        diff --git a/templates/green/html/user/favorites.html b/templates/green/html/user/favorites.html index 985bec0..08ebb60 100644 --- a/templates/green/html/user/favorites.html +++ b/templates/green/html/user/favorites.html @@ -53,7 +53,7 @@ 更新时间 - 书签 + 操作 @@ -94,7 +94,7 @@ var bookShelfListHtml = ""; for (var i = 0; i < bookShelfList.length; i++) { var book = bookShelfList[i]; - bookShelfListHtml += (" \n" + + bookShelfListHtml += (" \n" + " \n" + " [" + book.catName + "]\n" + " \n" + @@ -109,7 +109,8 @@ " " + book.lastIndexUpdateTime + "\n" + " \n" + " \n" + - "继续阅读" + + "" + + "" + " \n" + " "); } @@ -160,5 +161,21 @@ }) } + + function removeFromBookShelf(bookId) { + + $.ajax({ + type: "delete", + url: "/user/removeFromBookShelf/" + bookId, + data: {}, + dataType: "json", + success: function (data) { + if (data.code == 200) { + $("#shelf" + bookId).remove(); + } + } + }); + + } diff --git a/templates/green/html/user/login.html b/templates/green/html/user/login.html index 2420d9c..e22aafc 100644 --- a/templates/green/html/user/login.html +++ b/templates/green/html/user/login.html @@ -17,7 +17,7 @@
        -

        +

        • diff --git a/templates/green/static/css/base.css b/templates/green/static/css/base.css index 1246d57..a340870 100644 --- a/templates/green/static/css/base.css +++ b/templates/green/static/css/base.css @@ -964,4 +964,14 @@ i.vip_b { .userBox { margin: 0 auto +} + + +.layui-elem-quote { + margin-bottom: 10px; + padding: 15px; + line-height: 1.8; + border-left: 5px solid #16b777; + border-radius: 0 2px 2px 0; + background-color: #fafafa; } \ No newline at end of file diff --git a/templates/green/static/favicon.ico b/templates/green/static/favicon.ico new file mode 100644 index 0000000..67a7463 Binary files /dev/null and b/templates/green/static/favicon.ico differ diff --git a/templates/green/static/images/logo.png b/templates/green/static/images/logo.png index d29bb7d..b407a0d 100644 Binary files a/templates/green/static/images/logo.png and b/templates/green/static/images/logo.png differ diff --git a/templates/green/static/javascript/bookdetail.js b/templates/green/static/javascript/bookdetail.js index 7e78c68..3f6acd3 100644 --- a/templates/green/static/javascript/bookdetail.js +++ b/templates/green/static/javascript/bookdetail.js @@ -91,7 +91,7 @@ }, SaveComment: function (cmtBId, cmtCId, cmtDetail) { if (!isLogin) { - layer.alert('请先登陆'); + layer.alert('请先登录'); return; } var cmtDetailTemp = cmtDetail.replace(/(^\s*)/g, ""); diff --git a/templates/orange/html/author/author_income.html b/templates/orange/html/author/author_income.html index 3e9cc3a..7c74aaf 100644 --- a/templates/orange/html/author/author_income.html +++ b/templates/orange/html/author/author_income.html @@ -176,7 +176,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); @@ -206,7 +206,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); diff --git a/templates/orange/html/author/author_income_detail.html b/templates/orange/html/author/author_income_detail.html index 9232f02..1ea36a8 100644 --- a/templates/orange/html/author/author_income_detail.html +++ b/templates/orange/html/author/author_income_detail.html @@ -182,7 +182,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); @@ -212,7 +212,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); diff --git a/templates/orange/html/author/index.html b/templates/orange/html/author/index.html index d9f89ba..c6e9b4e 100644 --- a/templates/orange/html/author/index.html +++ b/templates/orange/html/author/index.html @@ -231,7 +231,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); @@ -261,7 +261,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/author/index_list.html b/templates/orange/html/author/index_list.html index f2f6702..ad07954 100644 --- a/templates/orange/html/author/index_list.html +++ b/templates/orange/html/author/index_list.html @@ -213,7 +213,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); @@ -273,7 +273,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); @@ -311,7 +311,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/book/book_content.html b/templates/orange/html/book/book_content.html index ea960d8..8a6f8fa 100644 --- a/templates/orange/html/book/book_content.html +++ b/templates/orange/html/book/book_content.html @@ -380,7 +380,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/book/bookclass.html b/templates/orange/html/book/bookclass.html index a44cea9..afa7291 100644 --- a/templates/orange/html/book/bookclass.html +++ b/templates/orange/html/book/bookclass.html @@ -6,8 +6,8 @@ - - + + diff --git a/templates/orange/html/index.html b/templates/orange/html/index.html index 53f6627..1e48ad7 100644 --- a/templates/orange/html/index.html +++ b/templates/orange/html/index.html @@ -7,8 +7,8 @@ - - + + diff --git a/templates/orange/html/mobile/book/book_detail.html b/templates/orange/html/mobile/book/book_detail.html index 705646c..5e8edf9 100644 --- a/templates/orange/html/mobile/book/book_detail.html +++ b/templates/orange/html/mobile/book/book_detail.html @@ -274,7 +274,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/mobile/pay/index.html b/templates/orange/html/mobile/pay/index.html index 4d35510..f3bc6b8 100644 --- a/templates/orange/html/mobile/pay/index.html +++ b/templates/orange/html/mobile/pay/index.html @@ -182,7 +182,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/mobile/user/userinfo.html b/templates/orange/html/mobile/user/userinfo.html index aa66546..793fd9d 100644 --- a/templates/orange/html/mobile/user/userinfo.html +++ b/templates/orange/html/mobile/user/userinfo.html @@ -244,7 +244,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/pay/index.html b/templates/orange/html/pay/index.html index 7c9fa03..002a56b 100644 --- a/templates/orange/html/pay/index.html +++ b/templates/orange/html/pay/index.html @@ -99,7 +99,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/user/comment.html b/templates/orange/html/user/comment.html index 1c080cd..a6fa3b8 100644 --- a/templates/orange/html/user/comment.html +++ b/templates/orange/html/user/comment.html @@ -109,7 +109,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); diff --git a/templates/orange/html/user/favorites.html b/templates/orange/html/user/favorites.html index 2f82588..efd996a 100644 --- a/templates/orange/html/user/favorites.html +++ b/templates/orange/html/user/favorites.html @@ -144,7 +144,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); diff --git a/templates/orange/html/user/feedback.html b/templates/orange/html/user/feedback.html index f6134c0..c326990 100644 --- a/templates/orange/html/user/feedback.html +++ b/templates/orange/html/user/feedback.html @@ -74,7 +74,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/user/feedback_list.html b/templates/orange/html/user/feedback_list.html index d61710d..71afc5b 100644 --- a/templates/orange/html/user/feedback_list.html +++ b/templates/orange/html/user/feedback_list.html @@ -109,7 +109,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); diff --git a/templates/orange/html/user/login.html b/templates/orange/html/user/login.html index 2420d9c..e22aafc 100644 --- a/templates/orange/html/user/login.html +++ b/templates/orange/html/user/login.html @@ -17,7 +17,7 @@
          -

          +

          • diff --git a/templates/orange/html/user/read_history.html b/templates/orange/html/user/read_history.html index ba91b3e..1bc99dd 100644 --- a/templates/orange/html/user/read_history.html +++ b/templates/orange/html/user/read_history.html @@ -131,7 +131,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); }else { layer.alert(data.msg); diff --git a/templates/orange/html/user/set_name.html b/templates/orange/html/user/set_name.html index 40e8037..1a5e44f 100644 --- a/templates/orange/html/user/set_name.html +++ b/templates/orange/html/user/set_name.html @@ -61,7 +61,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); @@ -96,7 +96,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { $("#LabErr").html(data.msg); diff --git a/templates/orange/html/user/set_password.html b/templates/orange/html/user/set_password.html index 8c8d6fb..5a7af3a 100644 --- a/templates/orange/html/user/set_password.html +++ b/templates/orange/html/user/set_password.html @@ -72,7 +72,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { $("#LabErr").html(data.msg); diff --git a/templates/orange/html/user/set_sex.html b/templates/orange/html/user/set_sex.html index bc90bb7..8505107 100644 --- a/templates/orange/html/user/set_sex.html +++ b/templates/orange/html/user/set_sex.html @@ -58,7 +58,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); @@ -81,7 +81,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/user/setup.html b/templates/orange/html/user/setup.html index 0a94c66..cb4c2e0 100644 --- a/templates/orange/html/user/setup.html +++ b/templates/orange/html/user/setup.html @@ -79,7 +79,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); @@ -118,7 +118,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/html/user/userinfo.html b/templates/orange/html/user/userinfo.html index e0c1496..50c4452 100644 --- a/templates/orange/html/user/userinfo.html +++ b/templates/orange/html/user/userinfo.html @@ -100,7 +100,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/static/javascript/bookdetail.js b/templates/orange/static/javascript/bookdetail.js index 8d5f66b..3e0a68d 100644 --- a/templates/orange/static/javascript/bookdetail.js +++ b/templates/orange/static/javascript/bookdetail.js @@ -35,7 +35,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); @@ -92,7 +92,7 @@ }, SaveComment: function (cmtBId, cmtCId, cmtDetail) { if(!isLogin){ - layer.alert('请先登陆'); + layer.alert('请先登录'); return; } var cmtDetailTemp = cmtDetail.replace(/(^\s*)/g, ""); @@ -121,7 +121,7 @@ } else if (data.code == 1001) { //未登录 - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } else { layer.alert(data.msg); diff --git a/templates/orange/static/mobile/js/common.js b/templates/orange/static/mobile/js/common.js index bc6a3ea..b8b3da1 100644 --- a/templates/orange/static/mobile/js/common.js +++ b/templates/orange/static/mobile/js/common.js @@ -106,7 +106,7 @@ Array.prototype.remove = function (val) { var token = $.cookie('Authorization'); if (!token) { if (needLoginPath.indexOf(window.location.pathname) != -1) { - location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href); + location.href = '/user/login.html?originUrl=' + encodeURIComponent(location.href); } $(".user_link").html("登录注册");