201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-26 17:20:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

增加小说内容TXT文本存储方案（一行配置切换）

Browse Source
This commit is contained in:
xiaoyang 2021-08-17 19:55:24 +08:00
parent cbfd0b049f
commit 750c8dee02
12 changed files with 262 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
novel-common/src/main/java/com/java2nb/novel/core/utils/FileUtil.java
View File

@ -19,6 +19,7 @@ import java.util.Objects;
/**
* 文件操作工具类
*
* @author 11797
*/
@UtilityClass
@ -27,8 +28,8 @@ public class FileUtil {
/**
* 网络图片转本地
* */
public String network2Local(String picSrc,String picSavePath,String visitPrefix) {
*/
public String network2Local(String picSrc, String picSavePath, String visitPrefix) {
InputStream input = null;
OutputStream out = null;
try {
@ -53,45 +54,41 @@ public class FileUtil {
}
out.flush();
if( ImageIO.read(picFile) == null){
if (ImageIO.read(picFile) == null) {
picSrc = "/images/default.gif";
}
}catch (Exception e){
log.error(e.getMessage(),e);
} catch (Exception e) {
log.error(e.getMessage(), e);
picSrc = "/images/default.gif";
}finally {
if(input != null){
try {
input.close();
} catch (IOException e) {
log.error(e.getMessage(),e);
}finally {
if(out != null){
try {
out.close();
} catch (IOException e) {
log.error(e.getMessage(),e);
}
}
}
}
} finally {
closeStream(input, out);
}
return picSrc;
}
@SneakyThrows
private void closeStream(InputStream input, OutputStream out) {
if (input != null) {
input.close();
}
if (out != null) {
out.close();
}
}
/**
* 判断文件是否为图片
*
* @param file 需要判断的文件
* @return true:是图片，false:不是图片
* */
*/
@SneakyThrows
public boolean isImage(File file){
public boolean isImage(File file) {
BufferedImage bi = ImageIO.read(file);
@ -100,6 +97,27 @@ public class FileUtil {
}
public void writeContentToFile(String fileSavePath, String fileSrc, String content) {
OutputStream out = null;
try {
File file = new File(fileSavePath + fileSrc);
File parentFile = file.getParentFile();
if (!parentFile.exists()) {
parentFile.mkdirs();
}
out = new FileOutputStream(file);
out.write(content.getBytes());
byte[] b = new byte[4096];
out.flush();
} catch (Exception e) {
log.error(e.getMessage(), e);
throw new RuntimeException("文件写入失败");
} finally {
closeStream(null, out);
}
}
}

4
novel-common/src/main/resources/application-common-dev.yml
View File

@ -71,3 +71,7 @@ sharding:
txt:
save:
storage: db #存储介质，db：数据库，file：txt文本
path: /Users/xiongxiaoyang/books #txt小说文本保存路径

5
novel-common/src/main/resources/application-common-prod.yml
View File

@ -74,7 +74,10 @@ logging:
txt:
save:
storage: db #存储介质，db：数据库，file：txt文本
path: /Users/xiongxiaoyang/books #txt小说文本保存路径

2
novel-common/src/main/resources/mybatis/generatorConfig.xml
View File

@ -44,7 +44,7 @@
</javaClientGenerator>
<!--生成全部表tableName设为%-->
<table tableName="news"/>
<table tableName="book_index"/>
<!-- 指定数据库表 -->
<!--<table schema="jly" tableName="job_position" domainObjectName="JobPositionTest"/>-->

16
novel-crawl/src/main/java/com/java2nb/novel/service/BookContentService.java Normal file
View File

@ -0,0 +1,16 @@
package com.java2nb.novel.service;
import com.java2nb.novel.entity.BookContent;
import java.util.List;
public interface BookContentService {
void saveBookContent(List<BookContent> bookContentList,Long bookId);
void saveBookContent(BookContent bookContent,Long bookId);
void updateBookContent(BookContent bookContent,Long bookId);
}

40
novel-crawl/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -4,6 +4,7 @@ import com.java2nb.novel.entity.Book;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.mapper.*;
import com.java2nb.novel.service.BookContentService;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.utils.Constants;
import lombok.RequiredArgsConstructor;
@ -37,7 +38,7 @@ public class BookServiceImpl implements BookService {
private final CrawlBookIndexMapper bookIndexMapper;
private final BookContentMapper bookContentMapper;
private final BookContentService bookContentService;
@Override
@ -46,7 +47,7 @@ public class BookServiceImpl implements BookService {
return bookMapper.count(countFrom(BookDynamicSqlSupport.book).where(BookDynamicSqlSupport.bookName, isEqualTo(bookName))
.and(BookDynamicSqlSupport.authorName, isEqualTo(authorName))
.build()
.render(RenderingStrategies.MYBATIS3))>0;
.render(RenderingStrategies.MYBATIS3)) > 0;
}
@ -57,7 +58,7 @@ public class BookServiceImpl implements BookService {
.equalTo(sourceId)
.set(crawlBookId)
.equalTo(bookId)
.where(BookDynamicSqlSupport.id,isEqualTo(id))
.where(BookDynamicSqlSupport.id, isEqualTo(id))
.build()
.render(RenderingStrategies.MYBATIS3));
}
@ -74,9 +75,9 @@ public class BookServiceImpl implements BookService {
@Transactional(rollbackFor = Exception.class)
@Override
public void saveBookAndIndexAndContent(Book book, List<BookIndex> bookIndexList, List<BookContent> bookContentList) {
if(!queryIsExistByBookNameAndAuthorName(book.getBookName(),book.getAuthorName())) {
if (!queryIsExistByBookNameAndAuthorName(book.getBookName(), book.getAuthorName())) {
if(bookIndexList.size()>0) {
if (bookIndexList.size() > 0) {
//保存小说主表
@ -85,7 +86,7 @@ public class BookServiceImpl implements BookService {
//批量保存目录和内容
bookIndexMapper.insertMultiple(bookIndexList);
bookContentMapper.insertMultiple(bookContentList);
bookContentService.saveBookContent(bookContentList,book.getId());
}
}
@ -96,7 +97,7 @@ public class BookServiceImpl implements BookService {
@Override
public List<Book> queryNeedUpdateBook(Date startDate, int limit) {
List<Book> books = bookMapper.queryNeedUpdateBook(startDate, limit);
if(books.size()>0) {
if (books.size() > 0) {
//更新最后抓取时间为当前时间
bookMapper.updateCrawlLastTime(books, new Date());
}
@ -105,38 +106,33 @@ public class BookServiceImpl implements BookService {
@Override
public Map<Integer, BookIndex> queryExistBookIndexMap(Long bookId) {
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id,BookIndexDynamicSqlSupport.indexNum,BookIndexDynamicSqlSupport.indexName,BookIndexDynamicSqlSupport.wordCount)
List<BookIndex> bookIndexs = bookIndexMapper.selectMany(select(BookIndexDynamicSqlSupport.id, BookIndexDynamicSqlSupport.indexNum, BookIndexDynamicSqlSupport.indexName, BookIndexDynamicSqlSupport.wordCount)
.from(BookIndexDynamicSqlSupport.bookIndex)
.where(BookIndexDynamicSqlSupport.bookId,isEqualTo(bookId))
.where(BookIndexDynamicSqlSupport.bookId, isEqualTo(bookId))
.build()
.render(RenderingStrategies.MYBATIS3));
if (bookIndexs.size() > 0) {
return bookIndexs.stream().collect(Collectors.toMap(BookIndex::getIndexNum, Function.identity()));
return bookIndexs.stream().collect(Collectors.toMap(BookIndex::getIndexNum, Function.identity()));
}
return new HashMap<>(0);
}
@Transactional(rollbackFor = Exception.class)
@Override
public void updateBookAndIndexAndContent(Book book, List<BookIndex> bookIndexList, List<BookContent> bookContentList, Map<Integer, BookIndex> existBookIndexMap) {
public void updateBookAndIndexAndContent(Book book, List<BookIndex> bookIndexList, List<BookContent> bookContentList, Map<Integer, BookIndex> existBookIndexMap) {
for (int i = 0; i < bookIndexList.size(); i++) {
BookIndex bookIndex = bookIndexList.get(i);
BookContent bookContent = bookContentList.get(i);
if(!existBookIndexMap.containsKey(bookIndex.getIndexNum())) {
if (!existBookIndexMap.containsKey(bookIndex.getIndexNum())) {
//插入
bookIndexMapper.insertSelective(bookIndex);
bookContentMapper.insertSelective(bookContent);
}else{
bookContentService.saveBookContent(bookContent,book.getId());
} else {
//更新
bookIndexMapper.updateByPrimaryKeySelective(bookIndex);
bookContentMapper.update(update(BookContentDynamicSqlSupport.bookContent)
.set(BookContentDynamicSqlSupport.content)
.equalTo(bookContent.getContent())
.where(BookContentDynamicSqlSupport.indexId,isEqualTo(bookContent.getIndexId()))
.build()
.render(RenderingStrategies.MYBATIS3));
bookContentService.updateBookContent(bookContent,book.getId());
}
@ -145,7 +141,7 @@ public class BookServiceImpl implements BookService {
//更新小说主表
book.setBookName(null);
book.setAuthorName(null);
if(Constants.VISIT_COUNT_DEFAULT.equals(book.getVisitCount())) {
if (Constants.VISIT_COUNT_DEFAULT.equals(book.getVisitCount())) {
book.setVisitCount(null);
}
bookMapper.updateByPrimaryKeySelective(book);
@ -168,7 +164,7 @@ public class BookServiceImpl implements BookService {
.build()
.render(RenderingStrategies.MYBATIS3));
if(books.size()>0){
if (books.size() > 0) {
return books.get(0);
}

44
novel-crawl/src/main/java/com/java2nb/novel/service/impl/DbBookContentServiceImpl.java Normal file
View File

@ -0,0 +1,44 @@
package com.java2nb.novel.service.impl;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.mapper.BookContentDynamicSqlSupport;
import com.java2nb.novel.mapper.BookContentMapper;
import com.java2nb.novel.service.BookContentService;
import lombok.RequiredArgsConstructor;
import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.List;
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.update;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "txt.save", name = "storage", havingValue = "db")
public class DbBookContentServiceImpl implements BookContentService {
private final BookContentMapper bookContentMapper;
@Override
public void saveBookContent(List<BookContent> bookContentList,Long bookId) {
bookContentMapper.insertMultiple(bookContentList);
}
@Override
public void saveBookContent(BookContent bookContent,Long bookId) {
bookContentMapper.insertSelective(bookContent);
}
@Override
public void updateBookContent(BookContent bookContent,Long bookId) {
bookContentMapper.update(update(BookContentDynamicSqlSupport.bookContent)
.set(BookContentDynamicSqlSupport.content)
.equalTo(bookContent.getContent())
.where(BookContentDynamicSqlSupport.indexId,isEqualTo(bookContent.getIndexId()))
.build()
.render(RenderingStrategies.MYBATIS3));
}
}

36
novel-crawl/src/main/java/com/java2nb/novel/service/impl/FileBookContentServiceImpl.java Normal file
View File

@ -0,0 +1,36 @@
package com.java2nb.novel.service.impl;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.service.BookContentService;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "txt.save", name = "storage", havingValue = "file")
public class FileBookContentServiceImpl implements BookContentService {
@Value("${txt.save.path}")
private String fileSavePath;
@Override
public void saveBookContent(List<BookContent> bookContentList,Long bookId) {
bookContentList.forEach(bookContent -> saveBookContent(bookContent,bookId));
}
@Override
public void saveBookContent(BookContent bookContent,Long bookId) {
FileUtil.writeContentToFile(fileSavePath,"/"+bookId+"/"+bookContent.getIndexId()+".txt",bookContent.getContent());
}
@Override
public void updateBookContent(BookContent bookContent,Long bookId) {
FileUtil.writeContentToFile(fileSavePath,"/"+bookId+"/"+bookContent.getIndexId()+".txt",bookContent.getContent());
}
}

15
novel-front/src/main/java/com/java2nb/novel/page/PageController.java
View File

@ -5,16 +5,14 @@ import com.java2nb.novel.core.bean.PageBean;
import com.java2nb.novel.core.bean.UserDetails;
import com.java2nb.novel.core.utils.ThreadLocalUtil;
import com.java2nb.novel.entity.*;
import com.java2nb.novel.service.AuthorService;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.NewsService;
import com.java2nb.novel.service.UserService;
import com.java2nb.novel.service.*;
import com.java2nb.novel.vo.BookCommentVO;
import com.java2nb.novel.vo.BookSettingVO;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.PathVariable;
@ -36,6 +34,9 @@ import java.util.concurrent.ThreadPoolExecutor;
@Controller
public class PageController extends BaseController {
@Value("${txt.save.path}")
private String fileSavePath;
private final BookService bookService;
private final NewsService newsService;
@ -44,6 +45,8 @@ public class PageController extends BaseController {
private final UserService userService;
private final BookContentService bookContentService;
private final ThreadPoolExecutor threadPoolExecutor;
@ -174,7 +177,7 @@ public class PageController extends BaseController {
*/
@SneakyThrows
@RequestMapping("/book/{bookId}/{bookIndexId}.html")
public String indexList(@PathVariable("bookId") Long bookId, @PathVariable("bookIndexId") Long bookIndexId, HttpServletRequest request, Model model) {
public String bookContent(@PathVariable("bookId") Long bookId, @PathVariable("bookIndexId") Long bookIndexId, HttpServletRequest request, Model model) {
//加载小说基本信息线程
CompletableFuture<Book> bookCompletableFuture = CompletableFuture.supplyAsync(() -> {
//查询书籍
@ -210,7 +213,7 @@ public class PageController extends BaseController {
//加载小说内容信息线程
CompletableFuture<BookContent> bookContentCompletableFuture = CompletableFuture.supplyAsync(() -> {
//查询内容
BookContent bookContent = bookService.queryBookContent(bookIndexId);
BookContent bookContent = bookContentService.queryBookContent(bookId, bookIndexId);
log.debug("加载小说内容信息线程结束");
return bookContent;
}, threadPoolExecutor);

11
novel-front/src/main/java/com/java2nb/novel/service/BookContentService.java Normal file
View File

@ -0,0 +1,11 @@
package com.java2nb.novel.service;
import com.java2nb.novel.entity.BookContent;
import java.util.List;
public interface BookContentService {
BookContent queryBookContent(Long bookId, Long bookIndexId);
}

38
novel-front/src/main/java/com/java2nb/novel/service/impl/DbBookContentServiceImpl.java Normal file
View File

@ -0,0 +1,38 @@
package com.java2nb.novel.service.impl;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.mapper.BookContentDynamicSqlSupport;
import com.java2nb.novel.mapper.BookContentMapper;
import com.java2nb.novel.service.BookContentService;
import lombok.RequiredArgsConstructor;
import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.List;
import static com.java2nb.novel.mapper.BookContentDynamicSqlSupport.bookContent;
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.update;
import static org.mybatis.dynamic.sql.select.SelectDSL.select;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "txt.save", name = "storage", havingValue = "db")
public class DbBookContentServiceImpl implements BookContentService {
private final BookContentMapper bookContentMapper;
@Override
public BookContent queryBookContent(Long bookId, Long bookIndexId) {
SelectStatementProvider selectStatement = select(BookContentDynamicSqlSupport.id, BookContentDynamicSqlSupport.content)
.from(bookContent)
.where(BookContentDynamicSqlSupport.indexId, isEqualTo(bookIndexId))
.limit(1)
.build()
.render(RenderingStrategies.MYBATIS3);
return bookContentMapper.selectMany(selectStatement).get(0);
}
}

39
novel-front/src/main/java/com/java2nb/novel/service/impl/FileBookContentServiceImpl.java Normal file
View File

@ -0,0 +1,39 @@
package com.java2nb.novel.service.impl;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.entity.BookContent;
import com.java2nb.novel.service.BookContentService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.List;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "txt.save", name = "storage", havingValue = "file")
public class FileBookContentServiceImpl implements BookContentService {
@Value("${txt.save.path}")
private String fileSavePath;
@SneakyThrows
@Override
public BookContent queryBookContent(Long bookId, Long bookIndexId) {
BufferedReader in = new BufferedReader(new FileReader(fileSavePath + "/" + bookId + "/" + bookIndexId + ".txt"));
StringBuffer sb = new StringBuffer();
String str;
while ((str = in.readLine()) != null) {
sb.append(str);
}
in.close();
return new BookContent() {{
setIndexId(bookIndexId);
setContent(sb.toString());
}};
}
}