201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-26 17:20:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

引入shardingjdbc,将小说内容表分为10个小表

Browse Source
This commit is contained in:
xiongxiaoyang 2020-05-26 18:09:19 +08:00
parent 06bf848e99
commit f9f1e474ec
9 changed files with 336 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
novel-common/pom.xml
View File

@ -57,6 +57,22 @@
<version>${mybatis.version}</version>
</dependency>
<!-- 分库分表-->
<!-- sharding jdbc依赖 -->
<dependency>
<groupId>io.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-boot-starter</artifactId>
<version>${sharding.jdbc.version}</version>
</dependency>
<dependency>
<groupId>io.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-namespace</artifactId>
<version>${sharding.jdbc.version}</version>
</dependency>
<!--分页助手启动器-->
<dependency>
<groupId>com.github.pagehelper</groupId>

2
novel-common/src/main/java/com/java2nb/novel/mapper/BookContentDynamicSqlSupport.java
View File

@ -24,7 +24,7 @@ public final class BookContentDynamicSqlSupport {
public final SqlColumn<Long> indexId = column("index_id", JDBCType.BIGINT);
public final SqlColumn<String> content = column("content", JDBCType.LONGVARCHAR);
public final SqlColumn<String> content = column("content", JDBCType.VARCHAR);
public BookContent() {
super("book_content");

2
novel-common/src/main/java/com/java2nb/novel/mapper/BookContentMapper.java
View File

@ -63,7 +63,7 @@ public interface BookContentMapper {
@Results(id="BookContentResult", value = {
@Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
@Result(column="index_id", property="indexId", jdbcType=JdbcType.BIGINT),
@Result(column="content", property="content", jdbcType=JdbcType.LONGVARCHAR)
@Result(column="content", property="content", jdbcType=JdbcType.VARCHAR)
})
List<BookContent> selectMany(SelectStatementProvider selectStatement);

56
novel-common/src/main/resources/application-common-dev.yml
View File

@ -1,11 +1,57 @@
spring:
profiles:
include: [common]
datasource:
url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: test123456
driver-class-name: com.mysql.cj.jdbc.Driver
main:
allow-bean-definition-overriding: true
# datasource:
# url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
# username: root
# password: test123456
# driver-class-name: com.mysql.cj.jdbc.Driver
shardingsphere:
datasource:
names: master
####使用shardingJdbc时，
####所有的jdbcType都不能是LONGVARCHAR,否则会导致java.io.NotSerializableException: java.io.StringReader错误
##### 应该替换所有的 LONGVARCHAR 类型为VARCHAR
sharding:
jdbc:
datasource:
names: ds0 #,ds1
ds0:
type: com.zaxxer.hikari.HikariDataSource
driver-class-name: com.mysql.cj.jdbc.Driver
jdbc-url: jdbc:mysql://localhost:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: test123456
# ds1:
# type: com.alibaba.druid.pool.DruidDataSource
# driver-class-name: com.mysql.jdbc.Driver
# url: jdbc:mysql://localhost:3306/novel_plus2
# username: root
# password: test123456
config:
sharding:
props:
sql.show: true
tables:
book_content: #book_content表
key-generator-column-name: id #主键
actual-data-nodes: ds${0}.book_content${0..9} #数据节点
# database-strategy: #分库策略
# inline:
# sharding-column: book_id
# algorithm-expression: ds${book_id % 10}
table-strategy: #分表策略
inline:
shardingColumn: index_id
algorithm-expression: book_content${index_id % 10}
#Redis服务器IP
redis:
host: 127.0.0.1

2
novel-crawl/pom.xml
View File

@ -25,6 +25,8 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
</dependencies>
<build>

15
novel-front/pom.xml
View File

@ -43,24 +43,19 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>2.4.0</version>
<version>${aliyun-sdk-oss.version}</version>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.3.1</version>
<version>${commons-fileupload.version}</version>
</dependency>
<!--FastDfs-->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-mock</artifactId>
<version>2.0.8</version>
</dependency>
<dependency>
<groupId>com.github.tobato</groupId>
<artifactId>fastdfs-client</artifactId>
<version>1.26.1-RELEASE</version>
<version>${fastdfs-client.version}</version>
</dependency>
<dependency>
@ -69,6 +64,8 @@
</dependency>
<!--引入redisson分布式锁-->
<!-- <dependency>
<groupId>org.redisson</groupId>
@ -80,7 +77,7 @@
<dependency>
<groupId>com.alipay.sdk</groupId>
<artifactId>alipay-sdk-java</artifactId>
<version>4.9.153.ALL</version>
<version>${alipay-sdk-java.version}</version>
</dependency>
</dependencies>

5
pom.xml
View File

@ -40,6 +40,11 @@
<jest.version>6.3.1</jest.version>
<redis.version>1.4.1.RELEASE</redis.version>
<redisson.version>3.12.5</redisson.version>
<sharding.jdbc.version>3.0.0</sharding.jdbc.version>
<aliyun-sdk-oss.version>2.4.0</aliyun-sdk-oss.version>
<commons-fileupload.version>1.3.1</commons-fileupload.version>
<fastdfs-client.version>1.26.1-RELEASE</fastdfs-client.version>
<alipay-sdk-java.version>4.9.153.ALL</alipay-sdk-java.version>
</properties>
<dependencyManagement>

136
sql/20200526.sql Normal file
View File

@ -0,0 +1,136 @@
/*
Navicat MySQL Data Transfer
Source Server : localhost
Source Server Version : 50725
Source Host : localhost:3306
Source Database : novel_plus
Target Server Type : MYSQL
Target Server Version : 50725
File Encoding : 65001
Date: 2020-05-26 18:06:21
*/
SET FOREIGN_KEY_CHECKS=0;
-- ----------------------------
-- Table structure for book_content0
-- ----------------------------
DROP TABLE IF EXISTS `book_content0`;
CREATE TABLE `book_content0` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1155 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content1
-- ----------------------------
DROP TABLE IF EXISTS `book_content1`;
CREATE TABLE `book_content1` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=406 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content2
-- ----------------------------
DROP TABLE IF EXISTS `book_content2`;
CREATE TABLE `book_content2` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1222 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content3
-- ----------------------------
DROP TABLE IF EXISTS `book_content3`;
CREATE TABLE `book_content3` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=410 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content4
-- ----------------------------
DROP TABLE IF EXISTS `book_content4`;
CREATE TABLE `book_content4` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1188 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content5
-- ----------------------------
DROP TABLE IF EXISTS `book_content5`;
CREATE TABLE `book_content5` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=416 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content6
-- ----------------------------
DROP TABLE IF EXISTS `book_content6`;
CREATE TABLE `book_content6` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1180 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content7
-- ----------------------------
DROP TABLE IF EXISTS `book_content7`;
CREATE TABLE `book_content7` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=404 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content8
-- ----------------------------
DROP TABLE IF EXISTS `book_content8`;
CREATE TABLE `book_content8` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1134 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content9
-- ----------------------------
DROP TABLE IF EXISTS `book_content9`;
CREATE TABLE `book_content9` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';

118
sql/novel_plus.sql
View File

@ -1691,4 +1691,122 @@ INSERT INTO `user_read_history` VALUES ('113', '1255060328322027520', '125467697
INSERT INTO `user_read_history` VALUES ('117', '1255060328322027520', '1254946661743603712', '1254946914001629184', '2020-04-30 19:37:09', '2020-04-30 19:37:09');
INSERT INTO `user_read_history` VALUES ('118', '1255060328322027520', '1254957312633352192', '3335449', '2020-04-30 19:37:36', '2020-04-30 19:37:36');
DROP TABLE IF EXISTS `book_content0`;
CREATE TABLE `book_content0` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1155 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content1
-- ----------------------------
DROP TABLE IF EXISTS `book_content1`;
CREATE TABLE `book_content1` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=406 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content2
-- ----------------------------
DROP TABLE IF EXISTS `book_content2`;
CREATE TABLE `book_content2` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1222 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content3
-- ----------------------------
DROP TABLE IF EXISTS `book_content3`;
CREATE TABLE `book_content3` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=410 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content4
-- ----------------------------
DROP TABLE IF EXISTS `book_content4`;
CREATE TABLE `book_content4` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1188 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content5
-- ----------------------------
DROP TABLE IF EXISTS `book_content5`;
CREATE TABLE `book_content5` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=416 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content6
-- ----------------------------
DROP TABLE IF EXISTS `book_content6`;
CREATE TABLE `book_content6` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1180 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content7
-- ----------------------------
DROP TABLE IF EXISTS `book_content7`;
CREATE TABLE `book_content7` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=404 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content8
-- ----------------------------
DROP TABLE IF EXISTS `book_content8`;
CREATE TABLE `book_content8` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1134 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
-- ----------------------------
-- Table structure for book_content9
-- ----------------------------
DROP TABLE IF EXISTS `book_content9`;
CREATE TABLE `book_content9` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',
`index_id` bigint(20) DEFAULT NULL COMMENT '目录ID',
`content` mediumtext COMMENT '小说章节内容',
PRIMARY KEY (`id`),
UNIQUE KEY `key_uq_indexId` (`index_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8mb4 COMMENT='小说内容表';
UPDATE `crawl_source` SET `source_name` = '书趣阁', `crawl_rule` = '{\n \"bookListUrl\": \"http://m.shuquge.com/sort/{catId}/0_{page}.html\",\n \"catIdRule\": {\n \"catId1\": \"1\",\n \"catId2\": \"2\",\n \"catId3\": \"3\",\n \"catId4\": \"4\",\n \"catId5\": \"7\",\n \"catId6\": \"6\",\n \"catId7\": \"8\"\n },\n \"bookIdPatten\": \"href=\\\"/s/(\\\\d+)\\\\.html\\\"\",\n \"pagePatten\": \"第(\\\\d+)/\\\\d+页\",\n \"totalPagePatten\": \"第\\\\d+/(\\\\d+)页\",\n \"bookDetailUrl\": \"http://m.shuquge.com/s/{bookId}.html\",\n \"bookNamePatten\": \"<a\\\\s+href=\\\"/s/\\\\d+\\\\.html\\\"><h2>([^/]+)</h2></a>\",\n \"authorNamePatten\": \"<p>作者：([^/]+)</p>\",\n \"picUrlPatten\": \"src=\\\"(http://www.shuquge.com/files/article/image/\\\\d+/\\\\d+/\\\\d+s\\\\.jpg)\\\"\",\n \"statusPatten\": \"<p>状态：([^/]+)</p>\",\n \"bookStatusRule\": {\n \"连载中\": 0,\n \"完本\": 1\n },\n \"descStart\": \"<div class=\\\"intro_info\\\">\",\n \"descEnd\": \"最新章节推荐地址\",\n \"bookIndexUrl\": \"http://www.shuquge.com/txt/{bookId}/index.html\",\n \"bookIndexStart\": \"<dt>《\",\n \"indexIdPatten\": \"<dd><a\\\\s+href=\\\"(\\\\d+)\\\\.html\\\">[^/]+</a></dd>\",\n \"indexNamePatten\": \"<dd><a\\\\s+href=\\\"\\\\d+\\\\.html\\\">([^/]+)</a></dd>\",\n \"bookContentUrl\": \"http://www.shuquge.com/txt/{bookId}/{indexId}.html\",\n \"contentStart\": \"<div id=\\\"content\\\" class=\\\"showtxt\\\">\",\n \"contentEnd\": \"http://www.shuquge.com\"\n}', `source_status` = 1, `create_time` = '2020-05-18 12:02:34', `update_time` = '2020-05-18 12:02:34' WHERE `id` = 4;