实现小说内容多种存储方式(txt、db..)并存

This commit is contained in:
xiongxiaoyang 2021-09-13 22:21:50 +08:00
parent bfe4d938fd
commit fc2ea40c6a
14 changed files with 138 additions and 105 deletions

3
doc/sql/20210913.sql Normal file
View File

@ -0,0 +1,3 @@
alter table book_index add column storage_type varchar(10) NOT NULL DEFAULT 'db' COMMENT '存储方式' after book_price ;

View File

@ -1841,3 +1841,6 @@ delete from sys_menu where menu_id = 202;
INSERT INTO crawl_source(`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES (16, 'i笔趣阁', '{\"bookListUrl\":\"http://m.ibiquge.net/xclass/{catId}/{page}.html\",\"catIdRule\":{\"catId1\":\"1\",\"catId2\":\"2\",\"catId3\":\"3\",\"catId4\":\"4\",\"catId5\":\"6\",\"catId6\":\"5\",\"catId7\":\"7\"},\"bookIdPatten\":\"href=\\\"/(\\\\d+_\\\\d+)/\\\"\",\"pagePatten\":\"value=\\\"(\\\\d+)/\\\\d+\\\"\",\"totalPagePatten\":\"value=\\\"\\\\d+/(\\\\d+)\\\"\",\"bookDetailUrl\":\"http://m.ibiquge.net/{bookId}/\",\"bookNamePatten\":\"<span class=\\\"title\\\">([^/]+)</span>\",\"authorNamePatten\":\"<a href=\\\"/author/\\\\d+/\\\">([^/]+)</a>\",\"picUrlPatten\":\"<img src=\\\"([^>]+)\\\"\\\\s+onerror=\\\"this.src=\",\"picUrlPrefix\":\"http://m.ibiquge.net\",\"statusPatten\":\">状态:([^/]+)</li>\",\"bookStatusRule\":{\"连载\":0,\"完结\":1},\"visitCountPatten\":\">点击:(\\\\d+)</li>\",\"descStart\":\"<p class=\\\"review\\\">\",\"descEnd\":\"</p>\",\"bookIndexUrl\":\"http://www.ibiquge.net/{bookId}/\",\"bookIndexStart\":\"正文</dt>\",\"indexIdPatten\":\"<a\\\\s+style=\\\"\\\"\\\\s+href=\\\"/\\\\d+_\\\\d+/(\\\\d+)\\\\.html\\\">[^/]+</a>\",\"indexNamePatten\":\"<a\\\\s+style=\\\"\\\"\\\\s+href=\\\"/\\\\d+_\\\\d+/\\\\d+\\\\.html\\\">([^/]+)</a>\",\"bookContentUrl\":\"http://www.ibiquge.net/{bookId}/{indexId}.html\",\"contentStart\":\"</p>\",\"contentEnd\":\"<div align=\\\"center\\\">\"}', 0, '2021-02-04 21:31:23', '2021-02-04 21:31:23'); INSERT INTO crawl_source(`id`, `source_name`, `crawl_rule`, `source_status`, `create_time`, `update_time`) VALUES (16, 'i笔趣阁', '{\"bookListUrl\":\"http://m.ibiquge.net/xclass/{catId}/{page}.html\",\"catIdRule\":{\"catId1\":\"1\",\"catId2\":\"2\",\"catId3\":\"3\",\"catId4\":\"4\",\"catId5\":\"6\",\"catId6\":\"5\",\"catId7\":\"7\"},\"bookIdPatten\":\"href=\\\"/(\\\\d+_\\\\d+)/\\\"\",\"pagePatten\":\"value=\\\"(\\\\d+)/\\\\d+\\\"\",\"totalPagePatten\":\"value=\\\"\\\\d+/(\\\\d+)\\\"\",\"bookDetailUrl\":\"http://m.ibiquge.net/{bookId}/\",\"bookNamePatten\":\"<span class=\\\"title\\\">([^/]+)</span>\",\"authorNamePatten\":\"<a href=\\\"/author/\\\\d+/\\\">([^/]+)</a>\",\"picUrlPatten\":\"<img src=\\\"([^>]+)\\\"\\\\s+onerror=\\\"this.src=\",\"picUrlPrefix\":\"http://m.ibiquge.net\",\"statusPatten\":\">状态:([^/]+)</li>\",\"bookStatusRule\":{\"连载\":0,\"完结\":1},\"visitCountPatten\":\">点击:(\\\\d+)</li>\",\"descStart\":\"<p class=\\\"review\\\">\",\"descEnd\":\"</p>\",\"bookIndexUrl\":\"http://www.ibiquge.net/{bookId}/\",\"bookIndexStart\":\"正文</dt>\",\"indexIdPatten\":\"<a\\\\s+style=\\\"\\\"\\\\s+href=\\\"/\\\\d+_\\\\d+/(\\\\d+)\\\\.html\\\">[^/]+</a>\",\"indexNamePatten\":\"<a\\\\s+style=\\\"\\\"\\\\s+href=\\\"/\\\\d+_\\\\d+/\\\\d+\\\\.html\\\">([^/]+)</a>\",\"bookContentUrl\":\"http://www.ibiquge.net/{bookId}/{indexId}.html\",\"contentStart\":\"</p>\",\"contentEnd\":\"<div align=\\\"center\\\">\"}', 0, '2021-02-04 21:31:23', '2021-02-04 21:31:23');
alter table news add column `read_count` BIGINT NOT NULL DEFAULT '0' COMMENT '阅读量' after content; alter table news add column `read_count` BIGINT NOT NULL DEFAULT '0' COMMENT '阅读量' after content;
alter table book_index add column storage_type varchar(10) NOT NULL DEFAULT 'db' COMMENT '存储方式' after book_price ;

View File

@ -25,6 +25,9 @@ public class BookIndex {
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
private Integer bookPrice; private Integer bookPrice;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private String storageType;
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
private Date createTime; private Date createTime;
@ -101,6 +104,16 @@ public class BookIndex {
this.bookPrice = bookPrice; this.bookPrice = bookPrice;
} }
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public String getStorageType() {
return storageType;
}
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public void setStorageType(String storageType) {
this.storageType = storageType == null ? null : storageType.trim();
}
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
public Date getCreateTime() { public Date getCreateTime() {
return createTime; return createTime;

View File

@ -31,6 +31,9 @@ public final class BookIndexDynamicSqlSupport {
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<Integer> bookPrice = bookIndex.bookPrice; public static final SqlColumn<Integer> bookPrice = bookIndex.bookPrice;
@Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<String> storageType = bookIndex.storageType;
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
public static final SqlColumn<Date> createTime = bookIndex.createTime; public static final SqlColumn<Date> createTime = bookIndex.createTime;
@ -53,6 +56,8 @@ public final class BookIndexDynamicSqlSupport {
public final SqlColumn<Integer> bookPrice = column("book_price", JDBCType.INTEGER); public final SqlColumn<Integer> bookPrice = column("book_price", JDBCType.INTEGER);
public final SqlColumn<String> storageType = column("storage_type", JDBCType.VARCHAR);
public final SqlColumn<Date> createTime = column("create_time", JDBCType.TIMESTAMP); public final SqlColumn<Date> createTime = column("create_time", JDBCType.TIMESTAMP);
public final SqlColumn<Date> updateTime = column("update_time", JDBCType.TIMESTAMP); public final SqlColumn<Date> updateTime = column("update_time", JDBCType.TIMESTAMP);

View File

@ -37,46 +37,47 @@ import org.mybatis.dynamic.sql.util.mybatis3.MyBatis3Utils;
@Mapper @Mapper
public interface BookIndexMapper { public interface BookIndexMapper {
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
BasicColumn[] selectList = BasicColumn.columnList(id, bookId, indexNum, indexName, wordCount, isVip, bookPrice, createTime, updateTime); BasicColumn[] selectList = BasicColumn.columnList(id, bookId, indexNum, indexName, wordCount, isVip, bookPrice, storageType, createTime, updateTime);
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
@SelectProvider(type = SqlProviderAdapter.class, method = "select") @SelectProvider(type=SqlProviderAdapter.class, method="select")
long count(SelectStatementProvider selectStatement); long count(SelectStatementProvider selectStatement);
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
@DeleteProvider(type = SqlProviderAdapter.class, method = "delete") @DeleteProvider(type=SqlProviderAdapter.class, method="delete")
int delete(DeleteStatementProvider deleteStatement); int delete(DeleteStatementProvider deleteStatement);
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
@InsertProvider(type = SqlProviderAdapter.class, method = "insert") @InsertProvider(type=SqlProviderAdapter.class, method="insert")
int insert(InsertStatementProvider<BookIndex> insertStatement); int insert(InsertStatementProvider<BookIndex> insertStatement);
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
@InsertProvider(type = SqlProviderAdapter.class, method = "insertMultiple") @InsertProvider(type=SqlProviderAdapter.class, method="insertMultiple")
int insertMultiple(MultiRowInsertStatementProvider<BookIndex> multipleInsertStatement); int insertMultiple(MultiRowInsertStatementProvider<BookIndex> multipleInsertStatement);
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
@SelectProvider(type = SqlProviderAdapter.class, method = "select") @SelectProvider(type=SqlProviderAdapter.class, method="select")
@ResultMap("BookIndexResult") @ResultMap("BookIndexResult")
Optional<BookIndex> selectOne(SelectStatementProvider selectStatement); Optional<BookIndex> selectOne(SelectStatementProvider selectStatement);
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
@SelectProvider(type = SqlProviderAdapter.class, method = "select") @SelectProvider(type=SqlProviderAdapter.class, method="select")
@Results(id = "BookIndexResult", value = { @Results(id="BookIndexResult", value = {
@Result(column = "id", property = "id", jdbcType = JdbcType.BIGINT, id = true), @Result(column="id", property="id", jdbcType=JdbcType.BIGINT, id=true),
@Result(column = "book_id", property = "bookId", jdbcType = JdbcType.BIGINT), @Result(column="book_id", property="bookId", jdbcType=JdbcType.BIGINT),
@Result(column = "index_num", property = "indexNum", jdbcType = JdbcType.INTEGER), @Result(column="index_num", property="indexNum", jdbcType=JdbcType.INTEGER),
@Result(column = "index_name", property = "indexName", jdbcType = JdbcType.VARCHAR), @Result(column="index_name", property="indexName", jdbcType=JdbcType.VARCHAR),
@Result(column = "word_count", property = "wordCount", jdbcType = JdbcType.INTEGER), @Result(column="word_count", property="wordCount", jdbcType=JdbcType.INTEGER),
@Result(column = "is_vip", property = "isVip", jdbcType = JdbcType.TINYINT), @Result(column="is_vip", property="isVip", jdbcType=JdbcType.TINYINT),
@Result(column = "book_price", property = "bookPrice", jdbcType = JdbcType.INTEGER), @Result(column="book_price", property="bookPrice", jdbcType=JdbcType.INTEGER),
@Result(column = "create_time", property = "createTime", jdbcType = JdbcType.TIMESTAMP), @Result(column="storage_type", property="storageType", jdbcType=JdbcType.VARCHAR),
@Result(column = "update_time", property = "updateTime", jdbcType = JdbcType.TIMESTAMP) @Result(column="create_time", property="createTime", jdbcType=JdbcType.TIMESTAMP),
@Result(column="update_time", property="updateTime", jdbcType=JdbcType.TIMESTAMP)
}) })
List<BookIndex> selectMany(SelectStatementProvider selectStatement); List<BookIndex> selectMany(SelectStatementProvider selectStatement);
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
@UpdateProvider(type = SqlProviderAdapter.class, method = "update") @UpdateProvider(type=SqlProviderAdapter.class, method="update")
int update(UpdateStatementProvider updateStatement); int update(UpdateStatementProvider updateStatement);
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
@ -92,52 +93,55 @@ public interface BookIndexMapper {
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
default int deleteByPrimaryKey(Long id_) { default int deleteByPrimaryKey(Long id_) {
return delete(c -> return delete(c ->
c.where(id, isEqualTo(id_)) c.where(id, isEqualTo(id_))
); );
} }
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
default int insert(BookIndex record) { default int insert(BookIndex record) {
return MyBatis3Utils.insert(this::insert, record, bookIndex, c -> return MyBatis3Utils.insert(this::insert, record, bookIndex, c ->
c.map(id).toProperty("id") c.map(id).toProperty("id")
.map(bookId).toProperty("bookId") .map(bookId).toProperty("bookId")
.map(indexNum).toProperty("indexNum") .map(indexNum).toProperty("indexNum")
.map(indexName).toProperty("indexName") .map(indexName).toProperty("indexName")
.map(wordCount).toProperty("wordCount") .map(wordCount).toProperty("wordCount")
.map(isVip).toProperty("isVip") .map(isVip).toProperty("isVip")
.map(bookPrice).toProperty("bookPrice") .map(bookPrice).toProperty("bookPrice")
.map(createTime).toProperty("createTime") .map(storageType).toProperty("storageType")
.map(updateTime).toProperty("updateTime") .map(createTime).toProperty("createTime")
.map(updateTime).toProperty("updateTime")
); );
} }
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
default int insertMultiple(Collection<BookIndex> records) { default int insertMultiple(Collection<BookIndex> records) {
return MyBatis3Utils.insertMultiple(this::insertMultiple, records, bookIndex, c -> return MyBatis3Utils.insertMultiple(this::insertMultiple, records, bookIndex, c ->
c.map(id).toProperty("id") c.map(id).toProperty("id")
.map(bookId).toProperty("bookId") .map(bookId).toProperty("bookId")
.map(indexNum).toProperty("indexNum") .map(indexNum).toProperty("indexNum")
.map(indexName).toProperty("indexName") .map(indexName).toProperty("indexName")
.map(wordCount).toProperty("wordCount") .map(wordCount).toProperty("wordCount")
.map(isVip).toProperty("isVip") .map(isVip).toProperty("isVip")
.map(bookPrice).toProperty("bookPrice") .map(bookPrice).toProperty("bookPrice")
.map(createTime).toProperty("createTime") .map(storageType).toProperty("storageType")
.map(updateTime).toProperty("updateTime") .map(createTime).toProperty("createTime")
.map(updateTime).toProperty("updateTime")
); );
} }
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
default int insertSelective(BookIndex record) { default int insertSelective(BookIndex record) {
return MyBatis3Utils.insert(this::insert, record, bookIndex, c -> return MyBatis3Utils.insert(this::insert, record, bookIndex, c ->
c.map(id).toPropertyWhenPresent("id", record::getId) c.map(id).toPropertyWhenPresent("id", record::getId)
.map(bookId).toPropertyWhenPresent("bookId", record::getBookId) .map(bookId).toPropertyWhenPresent("bookId", record::getBookId)
.map(indexNum).toPropertyWhenPresent("indexNum", record::getIndexNum) .map(indexNum).toPropertyWhenPresent("indexNum", record::getIndexNum)
.map(indexName).toPropertyWhenPresent("indexName", record::getIndexName) .map(indexName).toPropertyWhenPresent("indexName", record::getIndexName)
.map(wordCount).toPropertyWhenPresent("wordCount", record::getWordCount) .map(wordCount).toPropertyWhenPresent("wordCount", record::getWordCount)
.map(isVip).toPropertyWhenPresent("isVip", record::getIsVip) .map(isVip).toPropertyWhenPresent("isVip", record::getIsVip)
.map(bookPrice).toPropertyWhenPresent("bookPrice", record::getBookPrice) .map(bookPrice).toPropertyWhenPresent("bookPrice", record::getBookPrice)
.map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime) .map(storageType).toPropertyWhenPresent("storageType", record::getStorageType)
.map(updateTime).toPropertyWhenPresent("updateTime", record::getUpdateTime) .map(createTime).toPropertyWhenPresent("createTime", record::getCreateTime)
.map(updateTime).toPropertyWhenPresent("updateTime", record::getUpdateTime)
); );
} }
@ -159,7 +163,7 @@ public interface BookIndexMapper {
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
default Optional<BookIndex> selectByPrimaryKey(Long id_) { default Optional<BookIndex> selectByPrimaryKey(Long id_) {
return selectOne(c -> return selectOne(c ->
c.where(id, isEqualTo(id_)) c.where(id, isEqualTo(id_))
); );
} }
@ -177,6 +181,7 @@ public interface BookIndexMapper {
.set(wordCount).equalTo(record::getWordCount) .set(wordCount).equalTo(record::getWordCount)
.set(isVip).equalTo(record::getIsVip) .set(isVip).equalTo(record::getIsVip)
.set(bookPrice).equalTo(record::getBookPrice) .set(bookPrice).equalTo(record::getBookPrice)
.set(storageType).equalTo(record::getStorageType)
.set(createTime).equalTo(record::getCreateTime) .set(createTime).equalTo(record::getCreateTime)
.set(updateTime).equalTo(record::getUpdateTime); .set(updateTime).equalTo(record::getUpdateTime);
} }
@ -190,6 +195,7 @@ public interface BookIndexMapper {
.set(wordCount).equalToWhenPresent(record::getWordCount) .set(wordCount).equalToWhenPresent(record::getWordCount)
.set(isVip).equalToWhenPresent(record::getIsVip) .set(isVip).equalToWhenPresent(record::getIsVip)
.set(bookPrice).equalToWhenPresent(record::getBookPrice) .set(bookPrice).equalToWhenPresent(record::getBookPrice)
.set(storageType).equalToWhenPresent(record::getStorageType)
.set(createTime).equalToWhenPresent(record::getCreateTime) .set(createTime).equalToWhenPresent(record::getCreateTime)
.set(updateTime).equalToWhenPresent(record::getUpdateTime); .set(updateTime).equalToWhenPresent(record::getUpdateTime);
} }
@ -197,30 +203,32 @@ public interface BookIndexMapper {
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
default int updateByPrimaryKey(BookIndex record) { default int updateByPrimaryKey(BookIndex record) {
return update(c -> return update(c ->
c.set(bookId).equalTo(record::getBookId) c.set(bookId).equalTo(record::getBookId)
.set(indexNum).equalTo(record::getIndexNum) .set(indexNum).equalTo(record::getIndexNum)
.set(indexName).equalTo(record::getIndexName) .set(indexName).equalTo(record::getIndexName)
.set(wordCount).equalTo(record::getWordCount) .set(wordCount).equalTo(record::getWordCount)
.set(isVip).equalTo(record::getIsVip) .set(isVip).equalTo(record::getIsVip)
.set(bookPrice).equalTo(record::getBookPrice) .set(bookPrice).equalTo(record::getBookPrice)
.set(createTime).equalTo(record::getCreateTime) .set(storageType).equalTo(record::getStorageType)
.set(updateTime).equalTo(record::getUpdateTime) .set(createTime).equalTo(record::getCreateTime)
.where(id, isEqualTo(record::getId)) .set(updateTime).equalTo(record::getUpdateTime)
.where(id, isEqualTo(record::getId))
); );
} }
@Generated("org.mybatis.generator.api.MyBatisGenerator") @Generated("org.mybatis.generator.api.MyBatisGenerator")
default int updateByPrimaryKeySelective(BookIndex record) { default int updateByPrimaryKeySelective(BookIndex record) {
return update(c -> return update(c ->
c.set(bookId).equalToWhenPresent(record::getBookId) c.set(bookId).equalToWhenPresent(record::getBookId)
.set(indexNum).equalToWhenPresent(record::getIndexNum) .set(indexNum).equalToWhenPresent(record::getIndexNum)
.set(indexName).equalToWhenPresent(record::getIndexName) .set(indexName).equalToWhenPresent(record::getIndexName)
.set(wordCount).equalToWhenPresent(record::getWordCount) .set(wordCount).equalToWhenPresent(record::getWordCount)
.set(isVip).equalToWhenPresent(record::getIsVip) .set(isVip).equalToWhenPresent(record::getIsVip)
.set(bookPrice).equalToWhenPresent(record::getBookPrice) .set(bookPrice).equalToWhenPresent(record::getBookPrice)
.set(createTime).equalToWhenPresent(record::getCreateTime) .set(storageType).equalToWhenPresent(record::getStorageType)
.set(updateTime).equalToWhenPresent(record::getUpdateTime) .set(createTime).equalToWhenPresent(record::getCreateTime)
.where(id, isEqualTo(record::getId)) .set(updateTime).equalToWhenPresent(record::getUpdateTime)
.where(id, isEqualTo(record::getId))
); );
} }
} }

View File

@ -9,6 +9,7 @@ import com.java2nb.novel.service.BookService;
import com.java2nb.novel.utils.Constants; import com.java2nb.novel.utils.Constants;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.mybatis.dynamic.sql.render.RenderingStrategies; import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
@ -38,7 +39,10 @@ public class BookServiceImpl implements BookService {
private final CrawlBookIndexMapper bookIndexMapper; private final CrawlBookIndexMapper bookIndexMapper;
private final BookContentService bookContentService; private final Map<String, BookContentService> bookContentServiceMap;
@Value("${content.save.storage}")
private String storageType;
@Override @Override
@ -85,8 +89,11 @@ public class BookServiceImpl implements BookService {
bookMapper.insertSelective(book); bookMapper.insertSelective(book);
//批量保存目录和内容 //批量保存目录和内容
bookIndexList.forEach(bookIndex -> {
bookIndex.setStorageType(storageType);
});
bookIndexMapper.insertMultiple(bookIndexList); bookIndexMapper.insertMultiple(bookIndexList);
bookContentService.saveBookContent(bookContentList,book.getId()); bookContentServiceMap.get(storageType).saveBookContent(bookContentList, book.getId());
} }
} }
@ -106,7 +113,7 @@ public class BookServiceImpl implements BookService {
@Override @Override
public Map<Integer, BookIndex> queryExistBookIndexMap(Long bookId) { public Map<Integer, BookIndex> queryExistBookIndexMap(Long bookId) {
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount) List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount, BookIndexDynamicSqlSupport.storageType)
.from(BookIndexDynamicSqlSupport.bookIndex) .from(BookIndexDynamicSqlSupport.bookIndex)
.where(BookIndexDynamicSqlSupport.bookId, isEqualTo(bookId)) .where(BookIndexDynamicSqlSupport.bookId, isEqualTo(bookId))
.build() .build()
@ -127,12 +134,13 @@ public class BookServiceImpl implements BookService {
if (!existBookIndexMap.containsKey(bookIndex.getIndexNum())) { if (!existBookIndexMap.containsKey(bookIndex.getIndexNum())) {
//插入 //插入
bookIndex.setStorageType(storageType);
bookIndexMapper.insertSelective(bookIndex); bookIndexMapper.insertSelective(bookIndex);
bookContentService.saveBookContent(bookContent,book.getId()); bookContentServiceMap.get(storageType).saveBookContent(bookContent, book.getId());
} else { } else {
//更新 //更新
bookIndexMapper.updateByPrimaryKeySelective(bookIndex); bookIndexMapper.updateByPrimaryKeySelective(bookIndex);
bookContentService.updateBookContent(bookContent,book.getId()); bookContentServiceMap.get(existBookIndexMap.get(bookIndex.getIndexNum()).getStorageType()).updateBookContent(bookContent, book.getId());
} }

View File

@ -149,7 +149,7 @@ public class CrawlServiceImpl implements CrawlService {
@Override @Override
public void addCrawlSingleTask(CrawlSingleTask singleTask) { public void addCrawlSingleTask(CrawlSingleTask singleTask) {
if(bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(),singleTask.getAuthorName())){ if (bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(), singleTask.getAuthorName())) {
throw new BusinessException(ResponseStatus.BOOK_EXISTS); throw new BusinessException(ResponseStatus.BOOK_EXISTS);
} }
@ -181,23 +181,23 @@ public class CrawlServiceImpl implements CrawlService {
@Override @Override
public CrawlSingleTask getCrawlSingleTask() { public CrawlSingleTask getCrawlSingleTask() {
List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns()) List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
.from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask) .from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
.where(CrawlSingleTaskDynamicSqlSupport.taskStatus,isEqualTo((byte)2)) .where(CrawlSingleTaskDynamicSqlSupport.taskStatus, isEqualTo((byte) 2))
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime) .orderBy(CrawlSingleTaskDynamicSqlSupport.createTime)
.limit(1) .limit(1)
.build() .build()
.render(RenderingStrategies.MYBATIS3)); .render(RenderingStrategies.MYBATIS3));
return list.size() > 0 ? list.get(0) : null; return list.size() > 0 ? list.get(0) : null;
} }
@Override @Override
public void updateCrawlSingleTask(CrawlSingleTask task, Byte status) { public void updateCrawlSingleTask(CrawlSingleTask task, Byte status) {
byte excCount = task.getExcCount(); byte excCount = task.getExcCount();
excCount+=1; excCount += 1;
task.setExcCount(excCount); task.setExcCount(excCount);
if(status == 1 || excCount == 5){ if (status == 1 || excCount == 5) {
//当采集成功或者采集次数等于5则更新采集最终状态并停止采集 //当采集成功或者采集次数等于5则更新采集最终状态并停止采集
task.setTaskStatus(status); task.setTaskStatus(status);
} }
@ -219,7 +219,7 @@ public class CrawlServiceImpl implements CrawlService {
try { try {
if(StringUtils.isNotBlank(ruleBean.getCatIdRule().get("catId" + catId))) { if (StringUtils.isNotBlank(ruleBean.getCatIdRule().get("catId" + catId))) {
//拼接分类URL //拼接分类URL
String catBookListUrl = ruleBean.getBookListUrl() String catBookListUrl = ruleBean.getBookListUrl()
.replace("{catId}", ruleBean.getCatIdRule().get("catId" + catId)) .replace("{catId}", ruleBean.getCatIdRule().get("catId" + catId))
@ -235,7 +235,7 @@ public class CrawlServiceImpl implements CrawlService {
//1.阻塞过程使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时 //1.阻塞过程使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时
//捕获中断异常InterruptedException来退出线程 //捕获中断异常InterruptedException来退出线程
//2.非阻塞过程中通过判断中断标志来退出线程 //2.非阻塞过程中通过判断中断标志来退出线程
if(Thread.currentThread().isInterrupted()){ if (Thread.currentThread().isInterrupted()) {
return; return;
} }
@ -262,8 +262,8 @@ public class CrawlServiceImpl implements CrawlService {
} }
} }
}catch (Exception e){ } catch (Exception e) {
log.error(e.getMessage(),e); log.error(e.getMessage(), e);
} }
page += 1; page += 1;
@ -278,7 +278,7 @@ public class CrawlServiceImpl implements CrawlService {
final AtomicBoolean parseResult = new AtomicBoolean(false); final AtomicBoolean parseResult = new AtomicBoolean(false);
CrawlParser.parseBook(ruleBean, bookId, book -> { CrawlParser.parseBook(ruleBean, bookId, book -> {
if(book.getBookName() == null || book.getAuthorName() == null){ if (book.getBookName() == null || book.getAuthorName() == null) {
return; return;
} }
//这里只做新书入库查询是否存在这本书 //这里只做新书入库查询是否存在这本书
@ -301,7 +301,7 @@ public class CrawlServiceImpl implements CrawlService {
book.setCrawlLastTime(new Date()); book.setCrawlLastTime(new Date());
book.setId(new IdWorker().nextId()); book.setId(new IdWorker().nextId());
//解析章节目录 //解析章节目录
CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0),chapter -> { CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0), chapter -> {
bookService.saveBookAndIndexAndContent(book, chapter.getBookIndexList(), chapter.getBookContentList()); bookService.saveBookAndIndexAndContent(book, chapter.getBookIndexList(), chapter.getBookContentList());
}); });

View File

@ -15,9 +15,8 @@ import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.update; import static org.mybatis.dynamic.sql.SqlBuilder.update;
@Service @Service(value = "db")
@RequiredArgsConstructor @RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "db")
public class DbBookContentServiceImpl implements BookContentService { public class DbBookContentServiceImpl implements BookContentService {
private final BookContentMapper bookContentMapper; private final BookContentMapper bookContentMapper;

View File

@ -10,9 +10,8 @@ import org.springframework.stereotype.Service;
import java.util.List; import java.util.List;
@Service @Service(value = "txt")
@RequiredArgsConstructor @RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "file")
public class FileBookContentServiceImpl implements BookContentService { public class FileBookContentServiceImpl implements BookContentService {
@Value("${content.save.path}") @Value("${content.save.path}")

View File

@ -37,10 +37,10 @@ public class BookController extends BaseController {
private final BookService bookService; private final BookService bookService;
private final BookContentService bookContentService;
private final RabbitTemplate rabbitTemplate; private final RabbitTemplate rabbitTemplate;
private final Map<String, BookContentService> bookContentServiceMap;
@Value("${spring.rabbitmq.enable}") @Value("${spring.rabbitmq.enable}")
private Integer enableMq; private Integer enableMq;
@ -130,7 +130,8 @@ public class BookController extends BaseController {
public ResultBean<Map<String, Object>> queryBookIndexAbout(Long bookId, Long lastBookIndexId) { public ResultBean<Map<String, Object>> queryBookIndexAbout(Long bookId, Long lastBookIndexId) {
Map<String, Object> data = new HashMap<>(2); Map<String, Object> data = new HashMap<>(2);
data.put("bookIndexCount", bookService.queryIndexCount(bookId)); data.put("bookIndexCount", bookService.queryIndexCount(bookId));
String lastBookContent = bookContentService.queryBookContent(bookId,lastBookIndexId).getContent(); BookIndex bookIndex = bookService.queryBookIndex(lastBookIndexId);
String lastBookContent = bookContentServiceMap.get(bookIndex.getStorageType()).queryBookContent(bookId,lastBookIndexId).getContent();
if (lastBookContent.length() > 42) { if (lastBookContent.length() > 42) {
lastBookContent = lastBookContent.substring(0, 42); lastBookContent = lastBookContent.substring(0, 42);
} }

View File

@ -45,10 +45,9 @@ public class PageController extends BaseController {
private final UserService userService; private final UserService userService;
private final BookContentService bookContentService;
private final ThreadPoolExecutor threadPoolExecutor; private final ThreadPoolExecutor threadPoolExecutor;
private final Map<String, BookContentService> bookContentServiceMap;
@RequestMapping("{url}.html") @RequestMapping("{url}.html")
public String module(@PathVariable("url") String url) { public String module(@PathVariable("url") String url) {
@ -210,10 +209,10 @@ public class PageController extends BaseController {
return nextBookIndexId; return nextBookIndexId;
}, threadPoolExecutor); }, threadPoolExecutor);
//加载小说内容信息线程 //加载小说内容信息线程该线程在加载小说章节信息线程执行完毕后才执行
CompletableFuture<BookContent> bookContentCompletableFuture = CompletableFuture.supplyAsync(() -> { CompletableFuture<BookContent> bookContentCompletableFuture = bookIndexCompletableFuture.thenApplyAsync((bookIndex) -> {
//查询内容 //查询内容
BookContent bookContent = bookContentService.queryBookContent(bookId, bookIndexId); BookContent bookContent = bookContentServiceMap.get(bookIndex.getStorageType()).queryBookContent(bookId, bookIndexId);
log.debug("加载小说内容信息线程结束"); log.debug("加载小说内容信息线程结束");
return bookContent; return bookContent;
}, threadPoolExecutor); }, threadPoolExecutor);

View File

@ -262,7 +262,7 @@ public class BookServiceImpl implements BookService {
@Override @Override
public BookIndex queryBookIndex(Long bookIndexId) { public BookIndex queryBookIndex(Long bookIndexId) {
SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.bookId, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount,BookIndexDynamicSqlSupport.bookPrice, BookIndexDynamicSqlSupport.updateTime, BookIndexDynamicSqlSupport.isVip) SelectStatementProvider selectStatement = select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.bookId, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount,BookIndexDynamicSqlSupport.bookPrice, BookIndexDynamicSqlSupport.updateTime, BookIndexDynamicSqlSupport.isVip,BookIndexDynamicSqlSupport.storageType)
.from(bookIndex) .from(bookIndex)
.where(BookIndexDynamicSqlSupport.id, isEqualTo(bookIndexId)) .where(BookIndexDynamicSqlSupport.id, isEqualTo(bookIndexId))
.build() .build()

View File

@ -18,9 +18,8 @@ import static org.mybatis.dynamic.sql.SqlBuilder.update;
import static org.mybatis.dynamic.sql.select.SelectDSL.select; import static org.mybatis.dynamic.sql.select.SelectDSL.select;
@Service @Service(value = "db")
@RequiredArgsConstructor @RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "db")
public class DbBookContentServiceImpl implements BookContentService { public class DbBookContentServiceImpl implements BookContentService {
private final BookContentMapper bookContentMapper; private final BookContentMapper bookContentMapper;

View File

@ -1,21 +1,17 @@
package com.java2nb.novel.service.impl; package com.java2nb.novel.service.impl;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.entity.BookContent; import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.service.BookContentService; import com.java2nb.novel.service.BookContentService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.FileReader; import java.io.FileReader;
import java.util.List;
@Service @Service(value = "txt")
@RequiredArgsConstructor @RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "file")
public class FileBookContentServiceImpl implements BookContentService { public class FileBookContentServiceImpl implements BookContentService {
@Value("${content.save.path}") @Value("${content.save.path}")