201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-27 01:30:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

1优化es导入策略，2增加默认图片，在读取网络图片失败时设置，防止一直失败重试

Browse Source
This commit is contained in:
xiongxiaoyang 2020-05-21 08:16:37 +08:00
parent 0144b77983
commit a13ea78c3f
5 changed files with 61 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
novel-common/src/main/java/com/java2nb/novel/core/utils/FileUtil.java
View File

@ -1,6 +1,7 @@
package com.java2nb.novel.core.utils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.Charsets;
import org.apache.http.client.utils.DateUtils;
import org.springframework.core.io.Resource;
@ -17,34 +18,60 @@ import java.util.Objects;
* 文件操作工具类
* @author 11797
*/
@Slf4j
public class FileUtil {
/**
* 网络图片转本地
* */
@SneakyThrows
public static String network2Local(String picSrc,String picSavePath,String visitPrefix) {
//本地图片保存
HttpHeaders headers = new HttpHeaders();
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
ResponseEntity<Resource> resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1.name()).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
InputStream input = Objects.requireNonNull(resEntity.getBody()).getInputStream();
Date currentDate = new Date();
picSrc = visitPrefix + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd") + "/"
+ UUIDUtil.getUUID32()
+ picSrc.substring(picSrc.lastIndexOf("."));
File picFile = new File(picSavePath + picSrc);
File parentFile = picFile.getParentFile();
if (!parentFile.exists()) {
parentFile.mkdirs();
InputStream input = null;
OutputStream out = null;
try {
//本地图片保存
HttpHeaders headers = new HttpHeaders();
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
ResponseEntity<Resource> resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1.name()).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
input = Objects.requireNonNull(resEntity.getBody()).getInputStream();
Date currentDate = new Date();
picSrc = visitPrefix + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd") + "/"
+ UUIDUtil.getUUID32()
+ picSrc.substring(picSrc.lastIndexOf("."));
File picFile = new File(picSavePath + picSrc);
File parentFile = picFile.getParentFile();
if (!parentFile.exists()) {
parentFile.mkdirs();
}
out = new FileOutputStream(picFile);
byte[] b = new byte[4096];
for (int n; (n = input.read(b)) != -1; ) {
out.write(b, 0, n);
}
}catch (Exception e){
log.error(e.getMessage(),e);
picSrc = "/images/default.gif";
}finally {
if(input != null){
try {
input.close();
} catch (IOException e) {
log.error(e.getMessage(),e);
}finally {
if(out != null){
try {
out.close();
} catch (IOException e) {
log.error(e.getMessage(),e);
}
}
}
}
}
OutputStream out = new FileOutputStream(picFile);
byte[] b = new byte[4096];
for (int n; (n = input.read(b)) != -1; ) {
out.write(b, 0, n);
}
out.close();
input.close();
return picSrc;
}

13
novel-front/src/main/java/com/java2nb/novel/core/schedule/BookToEsSchedule.java
View File

@ -43,13 +43,12 @@ public class BookToEsSchedule {
private boolean lock = false;
/**
* 5秒导入一次
* 1分钟导入一次
*/
@Scheduled(fixedRate = 1000 * 5)
@Scheduled(fixedRate = 1000 * 60)
public void saveToEs() {
if (!lock) {
lock = true;
Date currentDate = new Date();
try {
//查询需要更新的小说
Date lastDate = (Date) cacheService.getObject(CacheKey.ES_LAST_UPDATE_TIME);
@ -58,10 +57,9 @@ public class BookToEsSchedule {
}
long count ;
int page = 1;
do {
List<Book> books = bookService.queryBookByUpdateTimeByPage(lastDate, currentDate,page,100);
List<Book> books = bookService.queryBookByUpdateTimeByPage(lastDate,100);
for(Book book : books) {
//导入到ES
EsBookVO esBookVO = new EsBookVO();
@ -71,14 +69,15 @@ public class BookToEsSchedule {
jestClient.execute(action);
lastDate = book.getUpdateTime();
}
count = books.size();
page++;
}while (count == 100);
cacheService.setObject(CacheKey.ES_LAST_UPDATE_TIME, currentDate);
cacheService.setObject(CacheKey.ES_LAST_UPDATE_TIME, lastDate);
} catch (Exception e) {
log.error(e.getMessage(),e);

6
novel-front/src/main/java/com/java2nb/novel/service/BookService.java
View File

@ -231,10 +231,8 @@ public interface BookService {
/**
* 根据更新时间分页查询书籍列表
* @param startDate 开始时间，包括该时间
* @param endDate 结束时间，不包括该时间
* @param page 页码
* @param pageSize 每页数量
* @param limit 查询数量
* @return 书籍列表
* */
List<Book> queryBookByUpdateTimeByPage(Date startDate, Date endDate, int page, int pageSize);
List<Book> queryBookByUpdateTimeByPage(Date startDate, int limit);
}

8
novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -719,14 +719,14 @@ public class BookServiceImpl implements BookService {
}
@Override
public List<Book> queryBookByUpdateTimeByPage(Date startDate, Date endDate, int page, int pageSize) {
public List<Book> queryBookByUpdateTimeByPage(Date startDate, int limit) {
PageHelper.startPage(page, pageSize);
return bookMapper.selectMany(select(book.allColumns())
.from(book)
.where(updateTime, isGreaterThanOrEqualTo(startDate))
.and(updateTime, isLessThan(endDate))
.where(updateTime, isGreaterThan(startDate))
.orderBy(updateTime)
.limit(limit)
.build()
.render(RenderingStrategies.MYBATIS3));
}

2
novel-front/src/main/resources/mybatis/mapping/BookMapper.xml
View File

@ -55,7 +55,7 @@
<select id="queryNetworkPicBooks" resultType="com.java2nb.novel.entity.Book">
select
id,pic_url from book
where pic_url like 'http://%' or pic_url like 'https://%'
where pic_url like 'http%'
limit #{offset},#{limit}
</select>