201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-26 17:20:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

实现小说内容多种存储方式（txt、db..）并存

Browse Source
This commit is contained in:
xiongxiaoyang 2021-09-13 22:21:50 +08:00
parent bfe4d938fd
commit fc2ea40c6a
14 changed files with 138 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/sql/20210913.sql Normal file
View File

@ -0,0 +1,3 @@
alter table book_index add column storage_type varchar(10) NOT NULL DEFAULT 'db' COMMENT '存储方式' after book_price ;

3
doc/sql/novel_plus.sql
View File

@ -1841,3 +1841,6 @@ delete from sys_menu where menu_id = 202;
INSERT INTO crawl_source(`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES (16, 'i笔趣阁', '{\"bookListUrl\":\"http://m.ibiquge.net/xclass/{catId}/{page}.html\",\"catIdRule\":{\"catId1\":\"1\",\"catId2\":\"2\",\"catId3\":\"3\",\"catId4\":\"4\",\"catId5\":\"6\",\"catId6\":\"5\",\"catId7\":\"7\"},\"bookIdPatten\":\"href=\\\"/(\\\\d+_\\\\d+)/\\\"\",\"pagePatten\":\"value=\\\"(\\\\d+)/\\\\d+\\\"\",\"totalPagePatten\":\"value=\\\"\\\\d+/(\\\\d+)\\\"\",\"bookDetailUrl\":\"http://m.ibiquge.net/{bookId}/\",\"bookNamePatten\":\"<span class=\\\"title\\\">([^/]+)</span>\",\"authorNamePatten\":\"<a href=\\\"/author/\\\\d+/\\\">([^/]+)</a>\",\"picUrlPatten\":\"<img src=\\\"([^>]+)\\\"\\\\s+onerror=\\\"this.src=\",\"picUrlPrefix\":\"http://m.ibiquge.net\",\"statusPatten\":\">状态：([^/]+)</li>\",\"bookStatusRule\":{\"连载\":0,\"完结\":1},\"visitCountPatten\":\">点击：(\\\\d+)</li>\",\"descStart\":\"<p class=\\\"review\\\">\",\"descEnd\":\"</p>\",\"bookIndexUrl\":\"http://www.ibiquge.net/{bookId}/\",\"bookIndexStart\":\"正文</dt>\",\"indexIdPatten\":\"<a\\\\s+style=\\\"\\\"\\\\s+href=\\\"/\\\\d+_\\\\d+/(\\\\d+)\\\\.html\\\">[^/]+</a>\",\"indexNamePatten\":\"<a\\\\s+style=\\\"\\\"\\\\s+href=\\\"/\\\\d+_\\\\d+/\\\\d+\\\\.html\\\">([^/]+)</a>\",\"bookContentUrl\":\"http://www.ibiquge.net/{bookId}/{indexId}.html\",\"contentStart\":\"</p>\",\"contentEnd\":\"<div align=\\\"center\\\">\"}', 0, '2021-02-04 21:31:23', '2021-02-04 21:31:23');
alter table news add column `read_count` BIGINT NOT NULL DEFAULT '0' COMMENT '阅读量' after content;
alter table book_index add column storage_type varchar(10) NOT NULL DEFAULT 'db' COMMENT '存储方式' after book_price ;

13
novel-common/src/main/java/com/java2nb/novel/entity/BookIndex.java
View File

@ -25,6 +25,9 @@ public class BookIndex {
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private Integer bookPrice;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private String storageType;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private Date createTime;
@ -101,6 +104,16 @@ public class BookIndex {
this.bookPrice = bookPrice;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public String getStorageType() {
return storageType;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public void setStorageType(String storageType) {
this.storageType = storageType == null ? null : storageType.trim();
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public Date getCreateTime() {
return createTime;

5
novel-common/src/main/java/com/java2nb/novel/mapper/BookIndexDynamicSqlSupport.java
View File

@ -31,6 +31,9 @@ public final class BookIndexDynamicSqlSupport {
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<Integer> bookPrice = bookIndex.bookPrice;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<String> storageType = bookIndex.storageType;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<Date> createTime = bookIndex.createTime;
@ -53,6 +56,8 @@ public final class BookIndexDynamicSqlSupport {
public final SqlColumn<Integer> bookPrice = column("book_price", JDBCType.INTEGER);
public final SqlColumn<String> storageType = column("storage_type", JDBCType.VARCHAR);
public final SqlColumn<Date> createTime = column("create_time", JDBCType.TIMESTAMP);
public final SqlColumn<Date> updateTime = column("update_time", JDBCType.TIMESTAMP);

10
novel-common/src/main/java/com/java2nb/novel/mapper/BookIndexMapper.java
View File

@ -37,7 +37,7 @@ import org.mybatis.dynamic.sql.util.mybatis3.MyBatis3Utils;
@Mapper
public interface BookIndexMapper {
@Generated("org.mybatis.generator.api.MyBatisGenerator")
BasicColumn[] selectList = BasicColumn.columnList(id, bookId, indexNum, indexName, wordCount, isVip, bookPrice, createTime, updateTime);
BasicColumn[] selectList = BasicColumn.columnList(id, bookId, indexNum, indexName, wordCount, isVip, bookPrice, storageType, createTime, updateTime);
@Generated("org.mybatis.generator.api.MyBatisGenerator")
@SelectProvider(type=SqlProviderAdapter.class, method="select")
@ -70,6 +70,7 @@ public interface BookIndexMapper {
@Result(column="word_count", property="wordCount", jdbcType=JdbcType.INTEGER),
@Result(column="is_vip", property="isVip", jdbcType=JdbcType.TINYINT),
@Result(column="book_price", property="bookPrice", jdbcType=JdbcType.INTEGER),
@Result(column="storage_type", property="storageType", jdbcType=JdbcType.VARCHAR),
@Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP),
@Result(column="update_time", property="updateTime", jdbcType=JdbcType.TIMESTAMP)
})
@ -106,6 +107,7 @@ public interface BookIndexMapper {
.map(wordCount).toProperty("wordCount")
.map(isVip).toProperty("isVip")
.map(bookPrice).toProperty("bookPrice")
.map(storageType).toProperty("storageType")
.map(createTime).toProperty("createTime")
.map(updateTime).toProperty("updateTime")
);
@ -121,6 +123,7 @@ public interface BookIndexMapper {
.map(wordCount).toProperty("wordCount")
.map(isVip).toProperty("isVip")
.map(bookPrice).toProperty("bookPrice")
.map(storageType).toProperty("storageType")
.map(createTime).toProperty("createTime")
.map(updateTime).toProperty("updateTime")
);
@ -136,6 +139,7 @@ public interface BookIndexMapper {
.map(wordCount).toPropertyWhenPresent("wordCount", record::getWordCount)
.map(isVip).toPropertyWhenPresent("isVip", record::getIsVip)
.map(bookPrice).toPropertyWhenPresent("bookPrice", record::getBookPrice)
.map(storageType).toPropertyWhenPresent("storageType", record::getStorageType)
.map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime)
.map(updateTime).toPropertyWhenPresent("updateTime", record::getUpdateTime)
);
@ -177,6 +181,7 @@ public interface BookIndexMapper {
.set(wordCount).equalTo(record::getWordCount)
.set(isVip).equalTo(record::getIsVip)
.set(bookPrice).equalTo(record::getBookPrice)
.set(storageType).equalTo(record::getStorageType)
.set(createTime).equalTo(record::getCreateTime)
.set(updateTime).equalTo(record::getUpdateTime);
}
@ -190,6 +195,7 @@ public interface BookIndexMapper {
.set(wordCount).equalToWhenPresent(record::getWordCount)
.set(isVip).equalToWhenPresent(record::getIsVip)
.set(bookPrice).equalToWhenPresent(record::getBookPrice)
.set(storageType).equalToWhenPresent(record::getStorageType)
.set(createTime).equalToWhenPresent(record::getCreateTime)
.set(updateTime).equalToWhenPresent(record::getUpdateTime);
}
@ -203,6 +209,7 @@ public interface BookIndexMapper {
.set(wordCount).equalTo(record::getWordCount)
.set(isVip).equalTo(record::getIsVip)
.set(bookPrice).equalTo(record::getBookPrice)
.set(storageType).equalTo(record::getStorageType)
.set(createTime).equalTo(record::getCreateTime)
.set(updateTime).equalTo(record::getUpdateTime)
.where(id, isEqualTo(record::getId))
@ -218,6 +225,7 @@ public interface BookIndexMapper {
.set(wordCount).equalToWhenPresent(record::getWordCount)
.set(isVip).equalToWhenPresent(record::getIsVip)
.set(bookPrice).equalToWhenPresent(record::getBookPrice)
.set(storageType).equalToWhenPresent(record::getStorageType)
.set(createTime).equalToWhenPresent(record::getCreateTime)
.set(updateTime).equalToWhenPresent(record::getUpdateTime)
.where(id, isEqualTo(record::getId))

18
novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -9,6 +9,7 @@ import com.java2nb.novel.service.BookService;
import com.java2nb.novel.utils.Constants;
import lombok.RequiredArgsConstructor;
import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@ -38,7 +39,10 @@ public class BookServiceImpl implements BookService {
private final CrawlBookIndexMapper bookIndexMapper;
private final BookContentService bookContentService;
private final Map<String, BookContentService> bookContentServiceMap;
@Value("${content.save.storage}")
private String storageType;
@Override
@ -85,8 +89,11 @@ public class BookServiceImpl implements BookService {
bookMapper.insertSelective(book);
//批量保存目录和内容
bookIndexList.forEach(bookIndex -> {
bookIndex.setStorageType(storageType);
});
bookIndexMapper.insertMultiple(bookIndexList);
bookContentService.saveBookContent(bookContentList,book.getId());
bookContentServiceMap.get(storageType).saveBookContent(bookContentList, book.getId());
}
}
@ -106,7 +113,7 @@ public class BookServiceImpl implements BookService {
@Override
public Map<Integer, BookIndex> queryExistBookIndexMap(Long bookId) {
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount)
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount, BookIndexDynamicSqlSupport.storageType)
.from(BookIndexDynamicSqlSupport.bookIndex)
.where(BookIndexDynamicSqlSupport.bookId, isEqualTo(bookId))
.build()
@ -127,12 +134,13 @@ public class BookServiceImpl implements BookService {
if (!existBookIndexMap.containsKey(bookIndex.getIndexNum())) {
//插入
bookIndex.setStorageType(storageType);
bookIndexMapper.insertSelective(bookIndex);
bookContentService.saveBookContent(bookContent,book.getId());
bookContentServiceMap.get(storageType).saveBookContent(bookContent, book.getId());
} else {
//更新
bookIndexMapper.updateByPrimaryKeySelective(bookIndex);
bookContentService.updateBookContent(bookContent,book.getId());
bookContentServiceMap.get(existBookIndexMap.get(bookIndex.getIndexNum()).getStorageType()).updateBookContent(bookContent, book.getId());
}

3
novel-crawl/src/main/java/com/java2nb/novel/service/impl/DbBookContentServiceImpl.java
View File

@ -15,9 +15,8 @@ import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.update;
@Service
@Service(value = "db")
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "db")
public class DbBookContentServiceImpl implements BookContentService {
private final BookContentMapper bookContentMapper;

3
novel-crawl/src/main/java/com/java2nb/novel/service/impl/FileBookContentServiceImpl.java
View File

@ -10,9 +10,8 @@ import org.springframework.stereotype.Service;
import java.util.List;
@Service
@Service(value = "txt")
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "file")
public class FileBookContentServiceImpl implements BookContentService {
@Value("${content.save.path}")

7
novel-front/src/main/java/com/java2nb/novel/controller/BookController.java
View File

@ -37,10 +37,10 @@ public class BookController extends BaseController {
private final BookService bookService;
private final BookContentService bookContentService;
private final RabbitTemplate rabbitTemplate;
private final Map<String, BookContentService> bookContentServiceMap;
@Value("${spring.rabbitmq.enable}")
private Integer enableMq;
@ -130,7 +130,8 @@ public class BookController extends BaseController {
public ResultBean<Map<String, Object>> queryBookIndexAbout(Long bookId, Long lastBookIndexId) {
Map<String, Object> data = new HashMap<>(2);
data.put("bookIndexCount", bookService.queryIndexCount(bookId));
String lastBookContent = bookContentService.queryBookContent(bookId,lastBookIndexId).getContent();
BookIndex bookIndex = bookService.queryBookIndex(lastBookIndexId);
String lastBookContent = bookContentServiceMap.get(bookIndex.getStorageType()).queryBookContent(bookId,lastBookIndexId).getContent();
if (lastBookContent.length() > 42) {
lastBookContent = lastBookContent.substring(0, 42);
}

9
novel-front/src/main/java/com/java2nb/novel/page/PageController.java
View File

@ -45,10 +45,9 @@ public class PageController extends BaseController {
private final UserService userService;
private final BookContentService bookContentService;
private final ThreadPoolExecutor threadPoolExecutor;
private final Map<String, BookContentService> bookContentServiceMap;
@RequestMapping("{url}.html")
public String module(@PathVariable("url") String url) {
@ -210,10 +209,10 @@ public class PageController extends BaseController {
return nextBookIndexId;
}, threadPoolExecutor);
//加载小说内容信息线程
CompletableFuture<BookContent> bookContentCompletableFuture = CompletableFuture.supplyAsync(() -> {
//加载小说内容信息线程，该线程在加载小说章节信息线程执行完毕后才执行
CompletableFuture<BookContent> bookContentCompletableFuture = bookIndexCompletableFuture.thenApplyAsync((bookIndex) -> {
//查询内容
BookContent bookContent = bookContentService.queryBookContent(bookId, bookIndexId);
BookContent bookContent = bookContentServiceMap.get(bookIndex.getStorageType()).queryBookContent(bookId, bookIndexId);
log.debug("加载小说内容信息线程结束");
return bookContent;
}, threadPoolExecutor);

2
novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -262,7 +262,7 @@ public class BookServiceImpl implements BookService {
@Override
public BookIndex queryBookIndex(Long bookIndexId) {
SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.bookId, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount,BookIndexDynamicSqlSupport.bookPrice, BookIndexDynamicSqlSupport.updateTime, BookIndexDynamicSqlSupport.isVip)
SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.bookId, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount,BookIndexDynamicSqlSupport.bookPrice, BookIndexDynamicSqlSupport.updateTime, BookIndexDynamicSqlSupport.isVip,BookIndexDynamicSqlSupport.storageType)
.from(bookIndex)
.where(BookIndexDynamicSqlSupport.id, isEqualTo(bookIndexId))
.build()

3
novel-front/src/main/java/com/java2nb/novel/service/impl/DbBookContentServiceImpl.java
View File

@ -18,9 +18,8 @@ import static org.mybatis.dynamic.sql.SqlBuilder.update;
import static org.mybatis.dynamic.sql.select.SelectDSL.select;
@Service
@Service(value = "db")
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "db")
public class DbBookContentServiceImpl implements BookContentService {
private final BookContentMapper bookContentMapper;

6
novel-front/src/main/java/com/java2nb/novel/service/impl/FileBookContentServiceImpl.java
View File

@ -1,21 +1,17 @@
package com.java2nb.novel.service.impl;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.service.BookContentService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.List;
@Service
@Service(value = "txt")
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "file")
public class FileBookContentServiceImpl implements BookContentService {
@Value("${content.save.path}")