From aa55cde2d318815abeae506162816a50526f6238 Mon Sep 17 00:00:00 2001 From: xiongxiaoyang <773861846@qq.com> Date: Tue, 24 Dec 2019 11:59:28 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=9C=AC=E5=9C=B0=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E6=A8=A1=E5=BC=8F=E4=B8=8B=EF=BC=8C=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E5=B0=86=E7=BD=91=E7=BB=9C=E5=9B=BE=E7=89=87=E8=BD=AC=E4=B8=BA?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E5=9B=BE=E7=89=87=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../schedule/Network2LocalPicSchedule.java | 58 +++++++++++++++++++ .../zinglizingli/books/mapper/BookMapper.java | 7 +++ .../books/service/BookService.java | 41 +++++-------- .../zinglizingli/common/utils/FileUtil.java | 50 ++++++++++++++++ .../src/main/resources/application-crawl.yml | 2 +- .../resources/mybatis/mapping/BookMapper.xml | 7 +++ 6 files changed, 138 insertions(+), 27 deletions(-) create mode 100644 novel-front/src/main/java/xyz/zinglizingli/books/core/schedule/Network2LocalPicSchedule.java create mode 100644 novel-front/src/main/java/xyz/zinglizingli/common/utils/FileUtil.java diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/core/schedule/Network2LocalPicSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/books/core/schedule/Network2LocalPicSchedule.java new file mode 100644 index 0000000..d829d1b --- /dev/null +++ b/novel-front/src/main/java/xyz/zinglizingli/books/core/schedule/Network2LocalPicSchedule.java @@ -0,0 +1,58 @@ +package xyz.zinglizingli.books.core.schedule; + + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import xyz.zinglizingli.books.core.enums.PicSaveType; +import xyz.zinglizingli.books.po.Book; +import xyz.zinglizingli.books.service.BookService; +import xyz.zinglizingli.common.utils.FileUtil; + +import java.util.List; + +/** + * 定时转换网络图片为本地图片 + * + * @author 11797 + */ +@ConditionalOnProperty(prefix = "pic.save",name = "type",havingValue = "2") +@Service +@RequiredArgsConstructor +@Slf4j +public class Network2LocalPicSchedule { + + private final BookService bookService; + + @Value("${pic.save.type}") + private Integer picSaveType; + + @Value("${pic.save.path}") + private String picSavePath; + + /** + * 10分钟转一次 + */ + @Scheduled(fixedRate = 1000 * 60 * 10) + public void trans() { + + log.info("Network2LocalPicSchedule。。。。。。。。。。。。"); + + + Integer offset = 0, limit = 100; + List networkPicBooks; + do { + networkPicBooks = bookService.queryNetworkPicBooks(limit, offset); + for (Book book : networkPicBooks) { + bookService.updateBook(book, book.getId()); + } + offset += limit; + } while (networkPicBooks.size() > 0); + + + } +} diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java index edee506..f2ed718 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/mapper/BookMapper.java @@ -46,4 +46,11 @@ public interface BookMapper { * 清理无效书籍(1个月前更新的无章节书籍) * */ void clearInvilidBook(); + + /** + * 查询网络图片的小说 + * + * @param limit + * @param offset*/ + List queryNetworkPicBooks(@Param("limit") Integer limit,@Param("offset") Integer offset); } \ No newline at end of file diff --git a/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java b/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java index 372e863..5cc9c82 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java +++ b/novel-front/src/main/java/xyz/zinglizingli/books/service/BookService.java @@ -20,15 +20,13 @@ import xyz.zinglizingli.books.core.enums.PicSaveType; import xyz.zinglizingli.books.mapper.*; import xyz.zinglizingli.books.po.*; import xyz.zinglizingli.books.core.utils.Constants; +import xyz.zinglizingli.common.utils.FileUtil; import xyz.zinglizingli.common.utils.SpringUtil; import xyz.zinglizingli.common.utils.UUIDUtils; import xyz.zinglizingli.common.cache.CommonCacheUtil; import xyz.zinglizingli.common.utils.RestTemplateUtil; -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -140,32 +138,12 @@ public class BookService { /** * 更新书籍 * */ - private void updateBook(Book book, Long bookId) { + public void updateBook(Book book, Long bookId) { book.setId(bookId); String picSrc = book.getPicUrl(); if(picSaveType == PicSaveType.LOCAL.getValue() && StringUtils.isNotBlank(picSrc)){ try { - //本地图片保存 - HttpHeaders headers = new HttpHeaders(); - HttpEntity requestEntity = new HttpEntity<>(null, headers); - ResponseEntity resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); - InputStream input = Objects.requireNonNull(resEntity.getBody()).getInputStream(); - Date currentDate = new Date(); - picSrc = "/localPic/" + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd") - + UUIDUtils.getUUID32() - + picSrc.substring(picSrc.lastIndexOf(".")); - File picFile = new File(picSavePath + picSrc); - File parentFile = picFile.getParentFile(); - if (!parentFile.exists()) { - parentFile.mkdirs(); - } - OutputStream out = new FileOutputStream(picFile); - byte[] b = new byte[4096]; - for (int n; (n = input.read(b)) != -1; ) { - out.write(b, 0, n); - } - out.close(); - input.close(); + picSrc = FileUtil.network2Local(picSrc,picSavePath); book.setPicUrl(picSrc); }catch (Exception e){ log.error(e.getMessage(),e); @@ -175,6 +153,8 @@ public class BookService { bookMapper.updateByPrimaryKeySelective(book); } + + /** * 批量插入章节目录表和章节内容表（自动修复错误章节） * */ @@ -441,4 +421,13 @@ public class BookService { //清楚无效书籍 bookMapper.clearInvilidBook(); } + + /** + * 查询网络图片的小说 + * + * @param limit + * @param offset*/ + public List queryNetworkPicBooks(Integer limit, Integer offset) { + return bookMapper.queryNetworkPicBooks(limit,offset); + } } diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/utils/FileUtil.java b/novel-front/src/main/java/xyz/zinglizingli/common/utils/FileUtil.java new file mode 100644 index 0000000..978428c --- /dev/null +++ b/novel-front/src/main/java/xyz/zinglizingli/common/utils/FileUtil.java @@ -0,0 +1,50 @@ +package xyz.zinglizingli.common.utils; + +import org.apache.commons.codec.Charsets; +import org.apache.http.client.utils.DateUtils; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; + +import java.io.*; +import java.util.Date; +import java.util.Objects; + +/** + * 文件操作工具类 + * @author 11797 + */ +public class FileUtil { + + /** + * 网络图片转本地 + * */ + public static String network2Local(String picSrc, String picSavePath) throws IOException { + //本地图片保存 + HttpHeaders headers = new HttpHeaders(); + HttpEntity requestEntity = new HttpEntity<>(null, headers); + ResponseEntity resEntity = RestTemplateUtil.getInstance(Charsets.ISO_8859_1).exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); + InputStream input = Objects.requireNonNull(resEntity.getBody()).getInputStream(); + Date currentDate = new Date(); + picSrc = "/localPic/" + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd") + + UUIDUtils.getUUID32() + + picSrc.substring(picSrc.lastIndexOf(".")); + File picFile = new File(picSavePath + picSrc); + File parentFile = picFile.getParentFile(); + if (!parentFile.exists()) { + parentFile.mkdirs(); + } + OutputStream out = new FileOutputStream(picFile); + byte[] b = new byte[4096]; + for (int n; (n = input.read(b)) != -1; ) { + out.write(b, 0, n); + } + out.close(); + input.close(); + return picSrc; + } + + +} diff --git a/novel-front/src/main/resources/application-crawl.yml b/novel-front/src/main/resources/application-crawl.yml index 545e354..7e72483 100644 --- a/novel-front/src/main/resources/application-crawl.yml +++ b/novel-front/src/main/resources/application-crawl.yml @@ -6,7 +6,7 @@ crawl: #抓取频率 period: 2000 -#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔, 3:顶点 更多网站解析中，敬请期待 +#爬取的网站名称类型 1：笔趣岛 ，2：笔趣塔, 3:顶点，默认顶点 更多网站解析中，敬请期待 biquta: crawlsource: enabled: true #是否开启此爬虫源 diff --git a/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml b/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml index 7b9fa6f..096da6c 100644 --- a/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml +++ b/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml @@ -395,4 +395,11 @@ where id not in (select book_id from book_index) and update_time DATE_SUB(curdate(),INTERVAL 10 DAY) + + \ No newline at end of file