201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-06-24 12:46:38 +00:00
Code Issues Packages Releases Wiki Activity

实现小说内容多种存储方式（txt、db..）并存

Browse Source
This commit is contained in:
xiongxiaoyang
2021-09-13 22:21:50 +08:00
parent bfe4d938fd
commit fc2ea40c6a
14 changed files with 138 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -9,6 +9,7 @@ import com.java2nb.novel.service.BookService;
import com.java2nb.novel.utils.Constants;
import lombok.RequiredArgsConstructor;
import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@ -38,7 +39,10 @@ public class BookServiceImpl implements BookService {
private final CrawlBookIndexMapper bookIndexMapper;
private final BookContentService bookContentService;
private final Map<String, BookContentService> bookContentServiceMap;
@Value("${content.save.storage}")
private String storageType;
@Override
@ -85,8 +89,11 @@ public class BookServiceImpl implements BookService {
bookMapper.insertSelective(book);
//批量保存目录和内容
bookIndexList.forEach(bookIndex -> {
bookIndex.setStorageType(storageType);
});
bookIndexMapper.insertMultiple(bookIndexList);
bookContentService.saveBookContent(bookContentList,book.getId());
bookContentServiceMap.get(storageType).saveBookContent(bookContentList, book.getId());
}
}
@ -106,7 +113,7 @@ public class BookServiceImpl implements BookService {
@Override
public Map<Integer, BookIndex> queryExistBookIndexMap(Long bookId) {
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount)
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount, BookIndexDynamicSqlSupport.storageType)
.from(BookIndexDynamicSqlSupport.bookIndex)
.where(BookIndexDynamicSqlSupport.bookId, isEqualTo(bookId))
.build()
@ -127,12 +134,13 @@ public class BookServiceImpl implements BookService {
if (!existBookIndexMap.containsKey(bookIndex.getIndexNum())) {
//插入
bookIndex.setStorageType(storageType);
bookIndexMapper.insertSelective(bookIndex);
bookContentService.saveBookContent(bookContent,book.getId());
bookContentServiceMap.get(storageType).saveBookContent(bookContent, book.getId());
} else {
//更新
bookIndexMapper.updateByPrimaryKeySelective(bookIndex);
bookContentService.updateBookContent(bookContent,book.getId());
bookContentServiceMap.get(existBookIndexMap.get(bookIndex.getIndexNum()).getStorageType()).updateBookContent(bookContent, book.getId());
}

28
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -149,7 +149,7 @@ public class CrawlServiceImpl implements CrawlService {
@Override
public void addCrawlSingleTask(CrawlSingleTask singleTask) {
if(bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(),singleTask.getAuthorName())){
if (bookService.queryIsExistByBookNameAndAuthorName(singleTask.getBookName(), singleTask.getAuthorName())) {
throw new BusinessException(ResponseStatus.BOOK_EXISTS);
}
@ -181,23 +181,23 @@ public class CrawlServiceImpl implements CrawlService {
@Override
public CrawlSingleTask getCrawlSingleTask() {
List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
List<CrawlSingleTask> list = crawlSingleTaskMapper.selectMany(select(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask.allColumns())
.from(CrawlSingleTaskDynamicSqlSupport.crawlSingleTask)
.where(CrawlSingleTaskDynamicSqlSupport.taskStatus,isEqualTo((byte)2))
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime)
.limit(1)
.where(CrawlSingleTaskDynamicSqlSupport.taskStatus, isEqualTo((byte) 2))
.orderBy(CrawlSingleTaskDynamicSqlSupport.createTime)
.limit(1)
.build()
.render(RenderingStrategies.MYBATIS3));
return list.size() > 0 ? list.get(0) : null;
return list.size() > 0 ? list.get(0) : null;
}
@Override
public void updateCrawlSingleTask(CrawlSingleTask task, Byte status) {
byte excCount = task.getExcCount();
excCount+=1;
excCount += 1;
task.setExcCount(excCount);
if(status == 1 || excCount == 5){
if (status == 1 || excCount == 5) {
//当采集成功或者采集次数等于5，则更新采集最终状态，并停止采集
task.setTaskStatus(status);
}
@ -219,7 +219,7 @@ public class CrawlServiceImpl implements CrawlService {
try {
if(StringUtils.isNotBlank(ruleBean.getCatIdRule().get("catId" + catId))) {
if (StringUtils.isNotBlank(ruleBean.getCatIdRule().get("catId" + catId))) {
//拼接分类URL
String catBookListUrl = ruleBean.getBookListUrl()
.replace("{catId}", ruleBean.getCatIdRule().get("catId" + catId))
@ -235,7 +235,7 @@ public class CrawlServiceImpl implements CrawlService {
//1.阻塞过程（使用了 sleep,同步锁的 wait,socket 中的 receiver,accept 等方法时）
//捕获中断异常InterruptedException来退出线程。
//2.非阻塞过程中通过判断中断标志来退出线程。
if(Thread.currentThread().isInterrupted()){
if (Thread.currentThread().isInterrupted()) {
return;
}
@ -262,8 +262,8 @@ public class CrawlServiceImpl implements CrawlService {
}
}
}catch (Exception e){
log.error(e.getMessage(),e);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
page += 1;
@ -278,7 +278,7 @@ public class CrawlServiceImpl implements CrawlService {
final AtomicBoolean parseResult = new AtomicBoolean(false);
CrawlParser.parseBook(ruleBean, bookId, book -> {
if(book.getBookName() == null || book.getAuthorName() == null){
if (book.getBookName() == null || book.getAuthorName() == null) {
return;
}
//这里只做新书入库，查询是否存在这本书
@ -301,7 +301,7 @@ public class CrawlServiceImpl implements CrawlService {
book.setCrawlLastTime(new Date());
book.setId(new IdWorker().nextId());
//解析章节目录
CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0),chapter -> {
CrawlParser.parseBookIndexAndContent(bookId, book, ruleBean, new HashMap<>(0), chapter -> {
bookService.saveBookAndIndexAndContent(book, chapter.getBookIndexList(), chapter.getBookContentList());
});

3
novel-crawl/src/main/java/com/java2nb/novel/service/impl/DbBookContentServiceImpl.java
View File

@ -15,9 +15,8 @@ import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.update;
@Service
@Service(value = "db")
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "db")
public class DbBookContentServiceImpl implements BookContentService {
private final BookContentMapper bookContentMapper;

3
novel-crawl/src/main/java/com/java2nb/novel/service/impl/FileBookContentServiceImpl.java
View File

@ -10,9 +10,8 @@ import org.springframework.stereotype.Service;
import java.util.List;
@Service
@Service(value = "txt")
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "content.save", name = "storage", havingValue = "file")
public class FileBookContentServiceImpl implements BookContentService {
@Value("${content.save.path}")